diff --git a/app/nullius/providers.py b/app/nullius/providers.py index 22c65bf..bf1d2da 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -16,6 +16,7 @@ import http.client import json import os +import re import secrets import socket import ssl @@ -211,6 +212,53 @@ def list_models(provider: str, env=None, *, opener=None) -> list[dict]: raise ValueError("Model list unavailable. Enter a model ID manually.") from None +# A model OpenRouter serves: its maker, a slash, its name (a variant after a colon: ``:free``). +_ROUTED = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._:-]*") +# A serving provider's slug, as ``--served-by`` takes it (``standard.relay.SLUG``). +_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def _endpoints(content: bytes) -> list[dict]: + """The serving providers of OpenRouter's endpoint list (``{"data": {"endpoints": [...]}}``), each + ``{slug, name, quantization}``: the slug its ``tag`` (a provider and its variant, ``deepinfra/fp8``), + else the provider's name lowercased with its spaces as dashes. ValueError when there is no list.""" + data = json.loads(content) + rows = ((data.get("data") or {}).get("endpoints") if isinstance(data, dict) and isinstance(data.get("data"), dict) else None) + if not isinstance(rows, list): + raise ValueError("missing endpoint list") + out = {} + for row in rows: + if not isinstance(row, dict): + continue + name = row.get("provider_name") if isinstance(row.get("provider_name"), str) else "" + tag = row.get("tag") if isinstance(row.get("tag"), str) else "" + slug = tag or re.sub(r"\s+", "-", name.strip().lower()) + if not slug or len(slug) > 128 or not _SLUG.fullmatch(slug) or slug in out: + continue + quant = row.get("quantization") if isinstance(row.get("quantization"), str) else "" + out[slug] = {"slug": slug, "name": (name or slug)[:128], "quantization": quant[:32]} + return sorted(out.values(), key=lambda e: (e["name"].casefold(), e["slug"])) + + +def list_endpoints(provider: str, model: str, env=None, *, opener=None) -> list[dict]: + """The serving providers OpenRouter lists for ``model`` (``GET /models/{author}/{slug}/endpoints``), for the + builder's Served by: asked as the model list is (its key when set, no redirect, nothing of the error echoed).""" + env = os.environ if env is None else env + provider, endpoint = split(provider) + if provider != "openrouter": + raise ValueError("Only OpenRouter routes a model to several providers.") + model = served_as(str(model or "").strip(), provider) + if not _ROUTED.fullmatch(model): + raise ValueError("Choose a model as OpenRouter names it, its maker first: maker/model.") + base, key = _where(provider, endpoint, env) + author, _, name = model.partition("/") + url = f"{base}/models/{urllib.parse.quote(author, safe='')}/{urllib.parse.quote(name, safe=':')}/endpoints" + try: + return _endpoints(_get(url, _bearer(key), opener)) + except (OSError, ValueError, TypeError, AttributeError, http.client.HTTPException): + raise ValueError("Provider list unavailable. Enter a provider's slug by hand.") from None + + # Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is # public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test. TESTED = ("openrouter", "openai", "anthropic", "custom") diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 56be6dd..2cd0065 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,9 +7,9 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] - PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] + PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force] PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed @@ -43,6 +43,9 @@ puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option`` sets one of the agent's or capability's options (``options`` in its definition); the rest keep the vendor's defaults, and the job's config records them all. +``--served-by [MODEL=]SLUG[,SLUG...]`` lets only those serving providers answer +a model on OpenRouter, in that order (``run.served_by``); each call's record +names the provider that served it. A run says each stage before its first unit, and each unit's end, as one ``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl`` (``nullius/progress.py``): what the run's page shows while it goes. @@ -525,6 +528,7 @@ def cmd_run(a): licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), + **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) @@ -596,11 +600,16 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) + jobs = options(a.job) + for c in jobs: + if c not in definitions.CONTROLS: + sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}") held, spill = licences(b) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, - env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images) + env=run_env(a.provider) if a.from_run else None, licences=held, + bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images) except run.Refused as e: print("refused:", e) return 2 @@ -694,6 +703,9 @@ def main(argv=None): p.add_argument("--constraint", action="append", metavar="KEY=VALUE", help="what the machine allows the run, repeatable: cpus, memory_mb, processes and cpu_set on its " "containers and, through a systemd scope, on its processes here; concurrency; gpu_ids (empty for none)") + p.add_argument("--served-by", action="append", metavar="[MODEL=]SLUG[,SLUG...]", + help="on OpenRouter, the serving providers that may answer a model, tried in this order and no " + "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") @@ -707,6 +719,10 @@ def main(argv=None): v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)") v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only") v.add_argument("--launch", help="the launch id the control jobs of every shard record") + v.add_argument("--bench-option", action="append", metavar="NAME=VALUE", + help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs") + v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR", + help="hold this job, a control run already over the whole dataset (a merged launch), to its bound instead of running it") v.add_argument("--remove-images", action="store_true", help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)") g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bafb447..9153d21 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + timeout: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -732,7 +733,9 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi an empty folder). ``hosts``: names it resolves (``--add-host``), the machine's own for an endpoint on its loopback. ``ledger``: the file of the run's spending cap, its folder mounted so - this relay books its calls where every other relay of the job does.""" + this relay books its calls where every other relay of the job does. + ``serving``: by model, the serving providers that may answer it, as the + unit's own relay is given them (``relay.Relay``).""" a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init", *capped(), *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")], @@ -752,8 +755,15 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] + # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's + sends = {model} | set(routes.values()) | {str(s).partition("=")[2] or str(s) for s in side_models} + for m, slugs in sorted((serving or {}).items()): + if m in sends: + a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] + if timeout: + a += ["--timeout", str(timeout)] return a @@ -1014,9 +1024,12 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True): + ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, + serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one + self.serving = dict(serving or {}) # by model, the serving providers that may answer it + self.timeout = timeout self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1042,7 +1055,8 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, + timeout=self.timeout) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 125c2b2..20b4f1f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -81,7 +81,7 @@ import re from typing import Callable, Dict, List, Optional, Tuple -from nullius.standard import ports +from nullius.standard import ports, relay from nullius import paths from nullius.catalogue import storage from . import names @@ -1035,6 +1035,25 @@ def control(bench: dict, which: str) -> Optional[dict]: "_kind": "control", "_ref": which, "_here": bench.get("_here", "")} +def comparable(orig: dict, model: str) -> bool: + """Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do.""" + c = orig.get("comparable", True) + return model in c if isinstance(c, list) else bool(c) + + +def upstream_name(orig: dict, model: str, provider: str) -> str: + """``model`` by the name the original's upstream knows, when given as ``provider`` lists it.""" + pre = (orig.get("providers") or {}).get(provider, "") + names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []} + return model if model in (orig.get("models") or []) else names.get(model, model) + + +def paper_models(orig: dict) -> List[str]: + """The original's models whose runs compare with the paper's.""" + c = orig.get("comparable", True) + return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c] + + def _leaderboard(d: dict) -> List[str]: """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the upstream's leaderboard fixes besides the ranked model, from its own submission @@ -1066,12 +1085,13 @@ def _leaderboard(d: dict) -> List[str]: def _setups(s) -> List[str]: """``setups``: {original: {summary, models, default, served_as?, providers, - side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s + side_models?, pin?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s ``models`` are the paper's, by the names the upstream's code knows them; ``side_models`` the ones its code asks besides the answerer (an extraction, a summary), sent as asked through served_as and the provider's prefix; ``providers`` the ones that serve them, each with the prefix its names - carry there; ``model`` needs ``original`` (its pipeline) and asks as one + carry there; ``pin`` {openrouter: {model: [provider slug]}}, the only + providers of each; ``model`` needs ``original`` (its pipeline) and asks as one of its models (``runs_as``).""" if s is None: return [] @@ -1090,7 +1110,7 @@ def _setups(s) -> List[str]: bad.append(f"{w}.summary: what runs, in a sentence") allowed = {"summary"} if k == "original": - allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"} + allowed |= {"models", "default", "served_as", "providers", "side_models", "pin", "units", "turn_seconds", "paper", "comparable", "reference_agent"} if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()): bad.append(f"{w}.reference_agent: the upstream agent's display name") models = x.get("models") @@ -1114,6 +1134,8 @@ def _setups(s) -> List[str]: pv = x.get("providers") if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()): bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}") + if "pin" in x: + bad += _pin(x["pin"], f"{w}.pin", pv if isinstance(pv, dict) else {}, set(models + side)) u = x.get("units") if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))): bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is") @@ -1122,8 +1144,11 @@ def _setups(s) -> List[str]: bad.append(f"{w}.turn_seconds: a whole number of seconds") if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()): bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase") - if "comparable" in x and not isinstance(x["comparable"], bool): - bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only") + c = x.get("comparable", True) + if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c) + or x.get("default") not in c): + bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; " + "or those of its models whose runs compare with the paper's, its default among them") else: allowed |= {"runs_as"} orig = s.get("original") if isinstance(s.get("original"), dict) else None @@ -1139,6 +1164,28 @@ def _setups(s) -> List[str]: return bad +PIN_PROVIDERS = ("openrouter",) + + +def _pin(pin, w: str, providers: dict, models: Optional[set] = None) -> List[str]: + """``pin``: {provider: {model: [slug]}} for a setup (``models`` given), {provider: [slug]} for a judge.""" + if not isinstance(pin, dict) or not pin: + return [f"{w}: {{openrouter: ...}}, the providers that alone serve it"] + bad = [] + for p, v in pin.items(): + if p not in PIN_PROVIDERS or (providers and p not in providers): + bad.append(f"{w}: a pin is OpenRouter's provider routing, on a provider the benchmark runs on ({p!r})") + continue + pins = v if models is not None else {"_": v} + if not isinstance(pins, dict) or not pins: + bad.append(f"{w}.{p}: {{model: [provider slug, ...]}}") + continue + if models is not None and not all(m in models for m in pins): + bad.append(f"{w}.{p}: pins only its models and side_models") + bad += [f"{w}.{p}: " + b.removeprefix("_: ") for b in relay.serving_problems(pins)] + return bad + + # Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder). SETTINGS_FILE = "upstream-settings.json" _DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$") @@ -1204,7 +1251,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple return None, f"{bench['label']} has no {which} setup" orig, x = s["original"], s[which] model = (model or "").strip() - side = [] + side, pins = [], {} if which == "original" and "models" not in orig: # any model, sent as the provider lists it if not model: return None, f"the original {bench['label']} needs a model: its leaderboard ranks models" @@ -1215,7 +1262,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple model = model or orig["default"] if model not in orig["models"]: return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): " - f"{model} is not one - choose the model setup for another") + f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else "")) if provider and provider not in orig["providers"]: return None, (f"the paper's models are not served by {provider}: the original runs on " + " or ".join(orig["providers"])) @@ -1225,13 +1272,15 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple sent = served(model) # the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent] + pins = {served(m): list(only) for m, only in ((orig.get("pin") or {}).get(provider) or {}).items() + if m == model or m in (orig.get("side_models") or [])} else: if not model: return None, f"the model setup needs a model: the one to put in {orig['default']}'s place" asked, sent = x["runs_as"], model units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"} - return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", - "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True), + return {"model": {"side_models": side}, **({"pins": pins} if pins else {}), "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", + "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model), "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which, "_here": bench.get("_here", "")}, "" @@ -1313,14 +1362,18 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") + if lim.get("run_again") is not None and not isinstance(lim["run_again"], bool): + bad.append("limits.run_again: false when a rate-limited unit is not run once more") a = lim.get("attempts") if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " "in a sentence - the driver is then sent attempt with setup, turn and answer") + if "gated" in d and not isinstance(d["gated"], bool): + bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept") s = d.get("score") or {} if s.get("kind") not in SCORES or not s.get("says"): bad.append(f"score: {{kind: one of {SCORES}, says}}") @@ -1332,6 +1385,8 @@ def problems(d: dict, kind: str) -> List[str]: j = s.get("judge") if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")): bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}") + elif j is not None and "pin" in j: + bad += _pin(j["pin"], "score.judge.pin", {}) sim = d.get("simulator") if sim is not None: if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"): diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index bc60de2..cd79086 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,8 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() + key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True, + timeout=r.call_seconds).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} @@ -230,7 +231,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k except (run._Ended, jsonl.Failed) as e: r.say(f" {tag} {unit}: {e}") try: - score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600) + score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds) except jsonl.Failed: score = {} if os.path.isdir(os.path.join(base, "agent")): diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 683d4e1..a4cd73c 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -57,9 +57,9 @@ def _text(content) -> str: def _usage_total(calls: List[dict]) -> dict: - ok = [c for c in calls if c.get("status") == 200] + ok = [c for c in calls if c.get("status") == 200 or c.get("reply")] u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok) - costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))] + costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float)) and c.get("cost_from") != "failed"] return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"), "cost": sum(costs) if costs else None, "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None), @@ -169,6 +169,9 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili reply = c.get("reply") or {} st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"), "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"} + if c.get("status") == 200 and reply.get("tool_calls"): + st["tool_calls"] = [{"tool_call_id": f"call-{len(steps) + 1}-{k + 1}", "function_name": t.get("name") or "", + "arguments": _arguments(t.get("arguments"))} for k, t in enumerate(reply["tool_calls"])] if reply.get("usage"): st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"], "cached_tokens": reply["usage"]["cached"]} @@ -183,6 +186,17 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}} +def _arguments(said) -> dict: + """A tool call's arguments as a dict; text that is no JSON object goes under ``raw``.""" + try: + got = json.loads(said) if isinstance(said, str) else said + except ValueError: + got = None + if said is None: + return {} + return got if isinstance(got, dict) else {"raw": said if isinstance(said, str) else json.dumps(said)} + + def _control(other: dict) -> dict: """A control's run says so, and who wrote the control: the runs pages read a standard job named ``nop`` or ``oracle`` as a control only by this.""" @@ -317,7 +331,7 @@ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tu if at is not None: turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True, content={"input": st.get("message")})) - elif agent or st.get("tool_calls"): + elif agent or (st.get("tool_calls") and not st.get("model_name")): spans.extend(_step_spans(i + 1, st, source)) if not spans and not turns: return None @@ -400,11 +414,74 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: return "full" if full else "partial" +def _named(name: str) -> str: + """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``DeepInfra`` reads + ``deepinfra``) and nothing else folded: ``Prov/A`` is not ``prov-a``, and a provider whose name differs from its + slug otherwise (``Google`` serving ``google-vertex``) reads as another. A run fetches no list of + providers, and this is all a name is matched by.""" + return re.sub(r"\s+", "-", name.strip().lower()) + + +def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> Optional[dict]: + """Who served a unit's answered calls (each record's ``reply.served_by``): the calls by serving provider and + those that named none; and, for each model the run pinned (``serving``), whether the pin held - true when + every answered call sent as that model names a provider that reads as one of its pinned slugs (``_named``, + a slug's first part), ``unknown`` when one names none or another name, None when it has no answered call. + A slug's endpoint variant (``/fp8``) is not in a reply's name: ``held`` says which provider answered, and + cannot tell one of its endpoints from another. None for a unit with no answered call in a run with no pin.""" + answered = [c for c in calls if c.get("status") == 200 and not (c.get("reply") or {}).get("error")] + by: Dict[str, int] = {} + unnamed = 0 + for c in answered: + name = (c.get("reply") or {}).get("served_by") + if isinstance(name, str) and name: + by[name] = by.get(name, 0) + 1 + else: + unnamed += 1 + if not answered and not serving: + return None + out = {"calls": dict(sorted(by.items())), "unnamed": unnamed} + if serving: + held = {} + for m, slugs in sorted(serving.items()): + mine = [c for c in answered if c.get("model") == m] + pinned = {str(s).split("/")[0].lower() for s in slugs} + held[m] = None if not mine else True if all( + _named(str((c.get("reply") or {}).get("served_by") or "")) in pinned for c in mine) else "unknown" + out["held"] = held + return out + + +def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[dict]: + """The job's ``served``: its trials' added up with ``also`` (the benchmark's own relay's, :func:`served`), a + pin held where it held in every one that called its model.""" + parts = [p for p in [(r.get("standard") or {}).get("served") for r in results] + [also] if p] + if not parts: + return None + calls: Dict[str, int] = {} + for p in parts: + for name, n in (p.get("calls") or {}).items(): + calls[name] = calls.get(name, 0) + n + out = {"calls": dict(sorted(calls.items())), "unnamed": sum(p.get("unnamed") or 0 for p in parts)} + models = sorted({m for p in parts for m in (p.get("held") or {})}) + if models: + out["held"] = {} + for m in models: + said = [p["held"][m] for p in parts if (p.get("held") or {}).get(m) is not None] + out["held"][m] = None if not said else True if all(v is True for v in said) else "unknown" + return out + + def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict: - """Write one unit's trial and return its result. ``environment``: the + image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), + serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None) -> dict: + """Write one unit's trial and return its result. ``serving``: the run's + pin of serving providers by model; ``standard.served`` says who served + ``served_calls`` (the unit's calls on every route of its relay: its + agent's, its judge's, its simulated user's; ``calls`` when not given) and + whether the pin held (:func:`served`). ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began @@ -426,13 +503,16 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]}, "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), - **({"environment": environment} if environment else {}), **({"image": image} if image else {})}} + **({"environment": environment} if environment else {}), **({"image": image} if image else {}), + **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score # benchmark a score and no pass or fail - the trial reads as Scored rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value} res["verifier_result"] = {"rewards": rewards} + if score.get("stopped"): + res["verifier_result"]["stopped"] = str(score["stopped"]) parts = score.get("parts") if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0: # a score over several parts (a conversation's questions): the job's mean weighs it by them @@ -451,9 +531,15 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) if trajectory is not None: _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1)) - timeline = trace_from_run(trajectory, calls, other, produce) - if timeline is not None: - _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + try: + timeline = trace_from_run(trajectory, calls, other, produce) + if timeline is not None: + _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + except Exception as e: + try: + _write(os.path.join(trial_dir, "agent", "trace.error.txt"), type(e).__name__ + "\n") + except OSError: + pass for name, text in (extra_logs or {}).items(): _write(os.path.join(trial_dir, "agent", name), text) _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1)) @@ -465,7 +551,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -477,7 +564,9 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str ``simulator``: the models that graded and that played the user, when the benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. - ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any.""" + ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any. + ``served_by``: by model, the serving providers the run let answer it + (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -488,7 +577,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), - **({"constraints": constraints} if constraints else {})}, + **({"constraints": constraints} if constraints else {}), + **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], @@ -544,14 +634,18 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} -def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): +def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None, + served: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, what was spent, whether it stopped the run, the units it cut off and - those never begun.""" + those never begun. ``served``: who served the benchmark's own relay's + calls (:func:`served`); the job's ``served`` adds its trials' to them.""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) + who = served_total(results, served) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, - **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1) + **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {}), + **({"served": who} if who else {})}, f, indent=1) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index c0d6636..955df60 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -9,8 +9,9 @@ the shards' with ``shard`` taken out and ``merged_from`` put in. What is merged must be one launch: the same launch id, benchmark and -version, dataset, agent or capability and model, and the same number of shards -- else it is refused, naming what differs, unless ``force``. Two trials of +version, dataset, agent or capability and model, the same serving providers +pinned for each model (``served_by``) and the same number of shards - else it +is refused, naming what differs, unless ``force``. Two trials of one unit are always refused, with the units named: one of them would be counted twice. So are shards whose lists of units (``run.units_file``) hold different units, or a shard of a list and one of none: the merged job @@ -18,7 +19,11 @@ comments, line endings and sha256 may differ from shard to shard, and the merged job records the first shard's list. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: -the shards absent, and the units a shard planned and has no trial for. +the shards absent, and the units a shard planned and has no trial for. A +trial whose record names another job, benchmark, version, control or agent +than its shard, or a job itself merged despite differences, is refused +unless ``force``. Shards run with different benchmark options keep them each +in ``merged_from``. Files are hard-linked into the merged job where the file system allows (a copy where it does not), so a merge of a thousand trials costs no second @@ -40,7 +45,13 @@ SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")), ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")), - ("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of"))) + ("backend", ("standard", "backend")), ("serving providers", ("standard", "served_by")), + ("number of shards", ("standard", "shard", "of"))) + + +LISTED = 1000 +OWN = (("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), + ("control", ("standard", "control", "name")), ("agent or capability", ("standard", "plugged"))) class Refused(Exception): @@ -73,6 +84,28 @@ def _listed(cfg: dict) -> Optional[tuple]: return tuple(sorted(str(u) for u in units)) if isinstance(units, list) else None +def _served(shards: List[dict], results: List[dict]) -> Optional[dict]: + """The merged job's ``served`` (``job.served_total``), made of each shard's whole one - its trials' and its + benchmark relay's calls, an ``unknown`` kept - else, for a shard that has none, of its trials'. A model the + shards pinned differently - merged with ``force`` - reads ``unknown``: the merged config keeps the first + shard's pin, which the others' calls were never held to.""" + parts = [] + for s in shards: + whole = (s.get("result") or {}).get("served") + if isinstance(whole, dict): + parts.append(whole) + else: + parts.append(job.served_total([r for r in results if (r.get("standard") or {}).get("shard_job") == s.get("name")])) + who = job.served_total([{"standard": {"served": p}} for p in parts if p]) + pins = [_at(s["config"], ("standard", "served_by")) or {} for s in shards] + pins = [p if isinstance(p, dict) else {} for p in pins] + differ = sorted(m for m in {m for p in pins for m in p} if len({json.dumps(p.get(m)) for p in pins}) > 1) + if differ: + who = who or {"calls": {}, "unnamed": 0} + who["held"] = dict(who.get("held") or {}, **{m: "unknown" for m in differ}) + return who + + def _trials(job_dir: str) -> List[str]: return sorted(n for n in os.listdir(job_dir) if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json"))) @@ -123,7 +156,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: + " (--force does not merge them)") differ = [] for label, path in SAME: - seen = {json.dumps(_at(s["config"], path)) for s in shards} + seen = {json.dumps(_at(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content if len(seen) > 1: differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards)) if differ and not force: @@ -131,12 +164,30 @@ def check(job_dirs: List[str], force: bool = False) -> dict: # every trial by its unit: two of one unit are never merged, force or not by_unit: Dict[str, List[tuple]] = {} names: Dict[str, List[str]] = {} + foreign = [] for s in shards: s["trials"] = _trials(s["dir"]) for t in s["trials"]: - unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t + res = _read(os.path.join(s["dir"], t, "result.json")) + unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) + wrong = [label for label, path in OWN if _at(res, path) != _at(s["config"], path)] + if _at(res, ("standard", "job")) != (s["config"].get("job_name") or s["name"]): + wrong.append("job") + if wrong: + foreign.append(f"{s['name']}/{t} names another {', '.join(wrong)} than its job") + inherited = [f"{s['name']}: {w}" for s in shards for w in _at(s["config"], ("standard", "merged_despite")) or []] + if inherited: + if not force: + raise Refused("a job given was itself merged despite differences - " + "; ".join(inherited[:5]) + + " (--force merges it anyway, and the merge says so)") + differ += inherited + if foreign: + shown = f"{len(foreign)} trial(s) not of the job they are in: " + "; ".join(foreign[:5]) + (" ..." if len(foreign) > 5 else "") + if not force: + raise Refused(shown + " (--force merges them anyway, and the merge says so)") + differ.append(shown) twice = {u: w for u, w in by_unit.items() if len(w) > 1} if twice: shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10]) @@ -151,9 +202,10 @@ def check(job_dirs: List[str], force: bool = False) -> dict: if len(of) == 1: n = of.pop() have = sorted(s["shard"].get("index") for s in shards if s["shard"]) - absent = [i for i in range(1, (n or 0) + 1) if i not in have] - if absent: - notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") + if isinstance(n, int) and n <= LISTED: + absent = [i for i in range(1, n + 1) if i not in have] + if absent: + notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") dup = sorted({i for i in have if have.count(i) > 1}) if dup: notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)") @@ -241,13 +293,21 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), "scored": len(scores), "mean_score": job.mean_score(results)}, - **({"budget": budget} if budget else {})}, f, indent=1) + **({"budget": budget} if budget else {}), + **({"served": who} if (who := _served(shards, results)) else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) std.pop("shard", None) + own = len({json.dumps(_at(s["config"], ("standard", "benchmark_options")), sort_keys=True) for s in shards}) > 1 + if own: + std.pop("benchmark_options", None) std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]), - **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards] + **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {}), + **({"benchmark_options": _at(s["config"], ("standard", "benchmark_options")) or {}} if own else {}), + **({"merged_from": _at(s["config"], ("standard", "merged_from"))} + if _at(s["config"], ("standard", "merged_from")) else {})} + for s in shards] if got["units_total"] is not None: std["units_total"] = got["units_total"] if got["differ"]: diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318..8b39142 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -30,6 +30,8 @@ pages pair them as a launch's controls); the units and leak checks talk to the driver directly and leave nothing behind. A control calls no model, so it needs no key - except a judged benchmark's, whose judge still grades. +``--bench-option`` reaches the checks' driver and each control's run; +``--job nop=DIR`` holds a merged launch of a control to its bound. A check that passes is recorded with its date, its counts and its job; one that fails is taken back, as ``conform`` takes back a port - what it reached kept @@ -41,6 +43,8 @@ """ from __future__ import annotations +import json +import math import os import re import shutil @@ -48,8 +52,7 @@ import time from typing import Callable, Dict, List, Optional -from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, run, shard as shards +from nullius.standard import definitions, install, job, jsonl, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -66,7 +69,8 @@ class _Driver: """The benchmark's driver, started as a run starts it, for the checks that talk to it directly (units, leak).""" - def __init__(self, bench: dict, ds: dict, work: str, licences=None): + def __init__(self, bench: dict, ds: dict, work: str, licences=None, options: Optional[dict] = None): + self.options = options if options is not None else options_for(bench, ds) self.up = install.upstream(bench) self.data = install.dataset(bench, ds, self.up) install.driver_packages(bench) # its driver.install, once: start() then finds it @@ -81,10 +85,15 @@ def start(self, n: int) -> jsonl.Proc: env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), - **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults + **run.option_env(self.options)}) return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() +def options_for(bench: dict, ds: dict, given: Optional[dict] = None, which=shutil.which) -> dict: + """The benchmark's options on ``ds`` for a control's run, at ``given`` or their defaults.""" + return run.resolve_bench_options(bench, ds["id"], given, "yours", which)[0] + + def units_check(driver: _Driver, ds: dict) -> dict: """Two drivers, one ``units`` call each: the same ids, unique, plain.""" got = [] @@ -209,44 +218,164 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str, provider: str, model: str, judge_model: Optional[str], env, concurrency: int, say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None, - licences=None, remove_images: bool = False) -> dict: + licences=None, options: Optional[dict] = None, job_dir: Optional[str] = None, + ids: Optional[List[str]] = None, remove_images: bool = False) -> dict: """Run one control over the dataset (or its first ``limit`` units, of - ``shard`` when given) and hold it to its bound: the pass rate, or the mean score.""" + ``shard`` when given) and hold it to its bound: the pass rate, or the mean score. + ``job_dir``: a control run already, held to the bound instead; it must cover ``ids``.""" other = definitions.control(bench, which) - job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, - jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), - env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images) - j = parse.parse_job(job_dir) - n = len(j.trials) - errored = sum(1 for t in j.trials if t.exception_type) + problems = [] + if job_dir: + job_dir = os.path.abspath(job_dir.rstrip("/")) + problems += job_problems(bench, ds, which, job_dir, ids or []) + cfg = _config(job_dir) + else: + job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, + jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), + say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, + licences=licences, bench_options=options, + remove_images=remove_images) + cfg = {} + field = FIELD[bench["score"]["kind"]] + results = [res for _, res in trials(job_dir)] + n = len(results) + errored = sum(1 for res in results if res.get("exception_info")) + values = [verdict(bench, res) for res in results if not res.get("exception_info")] + unscored = sum(1 for v in values if v is None) + values = [v for v in values if v is not None] + top = sum(1 for v in values if v >= 1.0) if bench["score"]["kind"] == "pass": - top = sum(1 for t in j.trials if t.reward == 1.0) value = top / n if n else 0.0 else: - scores = [t.score for t in j.trials if t.score is not None] - top = sum(1 for s in scores if s >= 1.0) - value = sum(scores) / len(scores) if scores else 0.0 + value = sum(values) / len(values) if values else 0.0 (bound, expected), = other["expect"].items() within = value <= expected if bound == "at_most" else value >= expected - problems = [] if not n: problems.append("no unit ran") if errored: problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials") + if unscored: + problems.append(f"{unscored} unit(s) ended with no {field} from 0 to 1 and no error: read the job's trials") if n and not within: problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}") rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected}, "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems} - if judge_model or (bench["score"].get("judge") or {}).get("model"): - rec["judge"] = judge_model or bench["score"]["judge"]["model"] + ran_with = _options_of(cfg) if cfg else (options or {}) + if ran_with: + rec["options"] = ran_with + judge = (cfg.get("standard") or {}).get("judge") if cfg else judge_model or (bench["score"].get("judge") or {}).get("model") + if judge: + rec["judge"] = judge return rec +FIELD = {"pass": "reward", "score": "score"} +UNITS_FAILED = "the units check failed" + + +def trials(job_dir: str) -> List[tuple]: + """(trial, its result.json) for each trial folder of a job; an unreadable record is {}.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if name.startswith(".") or not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + res = json.load(f) + except (OSError, ValueError): + res = {} + out.append((name, res if isinstance(res, dict) else {})) + return out + + +def verdict(bench: dict, res: dict) -> Optional[float]: + """The trial's value in the field its benchmark's score kind names, when it is a finite number from 0 to 1.""" + v = (((res.get("verifier_result") or {}).get("rewards")) or {}).get(FIELD[bench["score"]["kind"]]) + if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0.0 <= v <= 1.0: + return None + return float(v) + + +def _options_of(cfg: dict): + """The benchmark options a job ran with: one set, or each distinct set of its merged shards.""" + std = cfg.get("standard") or {} + if std.get("benchmark_options"): + return std["benchmark_options"] + sets, todo = [], list(std.get("merged_from") or []) + while todo: + m = todo.pop() + if m.get("benchmark_options") or not m.get("merged_from"): + sets.append(m.get("benchmark_options") or {}) + else: + todo += m.get("merged_from") or [] + distinct = {json.dumps(x, sort_keys=True): x for x in sets} + return [distinct[k] for k in sorted(distinct)] if any(sets) else [] + + +def _config(job_dir: str) -> dict: + try: + with open(os.path.join(job_dir, "config.json"), encoding="utf-8") as f: + got = json.load(f) + return got if isinstance(got, dict) else {} + except (OSError, ValueError): + return {} + + +def job_problems(bench: dict, ds: dict, which: str, job_dir: str, ids: List[str]) -> List[str]: + """Why ``job_dir`` is not one run of ``which`` over each unit of ``ids`` once.""" + cfg = _config(job_dir) + std = cfg.get("standard") or {} + if cfg.get("runner") != "standard": + return [f"{job_dir}: not a job of the Nullius standard"] + want = {"benchmark": bench["key"], "benchmark version": job._version(bench).get("benchmark_version"), + "control": which, "dataset": f"{bench['key']}/{ds['id']}"} + have = {"benchmark": std.get("benchmark"), "benchmark version": std.get("benchmark_version"), + "control": (std.get("control") or {}).get("name"), + "dataset": ((cfg.get("datasets") or [{}])[0] or {}).get("name")} + bad = [f"its {k} is {have[k]!r}, not {want[k]!r}" for k in want if have[k] != want[k]] + if std.get("merged_despite"): + bad.append("merged despite differences: " + "; ".join(std["merged_despite"])) + if (std.get("repeat") or 1) > 1: + bad.append(f"each unit ran {std['repeat']} times") + ran: Dict[str, int] = {} + shard_jobs = {m.get("job") for m in std.get("merged_from") or []} + alien: Dict[str, List[str]] = {} + for trial, res in trials(job_dir): + tstd = res.get("standard") if isinstance(res.get("standard"), dict) else {} + own = {"benchmark": tstd.get("benchmark"), "benchmark version": tstd.get("benchmark_version"), + "control": (tstd.get("control") or {}).get("name"), + "who ran it": (tstd.get("plugged"), (res.get("agent_info") or {}).get("name")), + "job": tstd.get("job")} + expect = {"benchmark": want["benchmark"], "benchmark version": want["benchmark version"], "control": which, + "who ran it": (which, which), "job": cfg.get("job_name")} + wrong = [k for k in expect if own[k] != expect[k]] + if shard_jobs and tstd.get("shard_job") not in shard_jobs: + wrong.append("shard") + for k in wrong: + alien.setdefault(k, []).append(trial) + u = str(tstd.get("unit") or trial) + ran[u] = ran.get(u, 0) + 1 + for k, names in alien.items(): + bad.append(f"{len(names)} trial(s) whose own record names another {k} than the job: {', '.join(names[:3])}") + known = set(ids) + missing = [u for u in ids if u not in ran] + extra = sorted(u for u in ran if u not in known) + twice = sorted(u for u, k in ran.items() if k > 1) + if missing: + bad.append(f"{len(missing)} of the dataset's {len(ids)} units have no trial: {', '.join(missing[:5])}" + (" ..." if len(missing) > 5 else "")) + if extra: + bad.append(f"{len(extra)} trial(s) of units the dataset does not have: {', '.join(extra[:5])}") + if twice: + bad.append(f"{len(twice)} unit(s) with more than one trial: {', '.join(twice[:5])}") + return [f"{os.path.basename(job_dir)}: {b}" for b in bad] + + def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs", provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None, env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print, shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None, + bench_options: Optional[dict] = None, jobs: Optional[Dict[str, str]] = None, remove_images: bool = False) -> Dict[str, dict]: """Every check on every dataset (or ``dataset``); with ``write``, what passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the @@ -254,13 +383,26 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = the units check all of them, and each record says which shard. ``remove_images``: each unit's own image is removed after it, so the oracle pulls it again (by default they stay). Returns - {dataset: {check: record}}, each record with ``ok`` and its ``problems``.""" + {dataset: {check: record}}, each record with ``ok`` and its ``problems``. + ``jobs``: {control: job directory}, held to its bound instead of run.""" env = env if env is not None else os.environ stamp = stamp or time.strftime(STAMP) + jobs = dict(jobs or {}) sets = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not sets: raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})] + if jobs: + if dataset is None and len(sets) > 1: + raise run.Refused("a control's job is a run on one dataset: name it (--dataset)") + if limit or (shard and shard[1] > 1): + raise run.Refused("a control's job is held over the whole dataset: no limit, no shard") + for c, d in jobs.items(): + if c not in declared: + raise run.Refused(f"{bench['label']} declares no {c} control") + if not os.path.isdir(d): + raise run.Refused(f"{d}: no such job") + opts = {ds["id"]: options_for(bench, ds, bench_options) for ds in sets} if not declared: say(f"{bench['label']} declares no controls: only its units and nothing else can be checked") got: Dict[str, dict] = {} @@ -270,11 +412,11 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = checks: Dict[str, dict] = {} try: try: - driver = _Driver(bench, ds, work, licences=licences) + driver = _Driver(bench, ds, work, licences=licences, options=opts[ds["id"]]) except install.InstallFailed as e: raise run.Refused(f"could not install what the proof needs: {e}") checks["units"] = units_check(driver, ds) - ids = checks["units"].pop("ids") + every = ids = checks["units"].pop("ids") if shard and shard[1] > 1: ids = [u for _, u in shards.select(ids, *shard)] ids = ids[:limit] if limit else ids @@ -285,10 +427,18 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = finally: shutil.rmtree(work, ignore_errors=True) for c in declared: - checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, - model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, - shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, - remove_images=remove_images) + if not checks["units"]["ok"] and not jobs.get(c): + checks[c] = {"ok": False, "units": 0, "source": definitions.control(bench, c)["source"], + "problems": [UNITS_FAILED + ": the control was not run"]} + else: + checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, + model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, + shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, + options=opts[ds["id"]], job_dir=jobs.get(c), ids=every, + remove_images=remove_images) + if not checks["units"]["ok"]: + checks[c]["problems"].append(UNITS_FAILED + ": a job's cover is held to the driver's ids, which did not pass") + checks[c]["ok"] = False _say(say, c, checks[c]) total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"] for name, r in checks.items(): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 9e397d1..b6a6a73 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -40,7 +40,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove +from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay HARNESS = "standard" # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's. @@ -49,6 +49,8 @@ MAX_CONCURRENT = 32 BACKENDS = definitions.names.BACKENDS _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") +# A pin's field, as the Review's Model row names it (served_by, --served-by) +SERVED_LABEL = {"model": "model", "simulator": "user simulator", "judge": "judge"} _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$") @@ -124,9 +126,10 @@ def _bench_page(b: dict) -> dict: # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], "reference_agent": x.get("reference_agent", ""), - "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), - **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), - "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} + "comparable": k == "original" and b["setups"]["original"].get("comparable", True) is not False, "units": (b["setups"]["original"].get("units") or b.get("units") or {}), + **({"models": definitions.paper_models(x), "default": x["default"], "providers": list(x["providers"]), + "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in definitions.paper_models(x)} + for p, pre in x["providers"].items()}} if "models" in x else {"any": True, "providers": list(x["providers"])})} for k, x in (b.get("setups") or {}).items() if k == "original"], # its own options (a search, a mode): each value with its line and what it needs; and the host requirements @@ -281,11 +284,17 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it names have no price in the user's list (``nullius.prices``): a run - with a cap is refused without one, and the Limits step says so first.""" + with a cap is refused without one, and the Limits step says so first. + + ``served_by``: on OpenRouter, the serving providers each model field + pins, ``{model|simulator|judge: [slug, ...]}``, checked as ``--served-by`` + is and compiled into it for each job that calls that model, by the name + the run sends it as.""" d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) - out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""} + out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "", + "served_models": {}} name = str(recipe.get("name") or "") if not _NAME.match(name): add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.") @@ -331,6 +340,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "Choose a model.") continue method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" + if method == "original": + picked = definitions.upstream_name(orig, picked, provider_) o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") @@ -407,8 +418,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()): add("model", "No paper models are listed for this benchmark; choose Any of your choice.") elif model_choice == "paper" and needs_model and orig.get("models"): - if recipe.get("model") not in orig["models"]: - add("model", "Choose one of the paper's listed models.") + if recipe.get("model") not in definitions.paper_models(orig): + add("model", "Choose one of the paper's listed models, or Any of your choice for another of the original's.") elif provider_ not in orig.get("providers", {}): add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") @@ -437,6 +448,20 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model + # the serving providers each model field pins, on OpenRouter (served_by: {model|simulator|judge: [slug, ...]}), + # checked as --served-by is (relay.serving_problems); an empty list pins nothing + roles = {"model": "the model", "simulator": "the user simulator's model", "judge": "the judge's model"} + pins = recipe.get("served_by") + served: Dict[str, List[str]] = {} + if pins not in (None, {}): + if not isinstance(pins, dict) or any(k not in roles for k in pins): + add("model", "Served by names the serving providers of the model, the user simulator's or the judge's.") + else: + served = {k: v for k, v in pins.items() if v not in (None, [], "")} + for problem in relay.serving_problems({roles[k]: v for k, v in served.items()}): + add("model", "Served by " + problem + ".") + if served and provider_ != "openrouter": + add("model", "Served by needs OpenRouter, the provider that routes a model to several: choose it, or leave Served by empty.") # the run's spending cap: an amount in USD, or none cap = recipe.get("max_usd") if cap in ("", None): @@ -495,6 +520,42 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, limits.of(t) except ValueError as e: add("where", f"{e}: fix it in Settings.") + judge_def, sim_def = (b.get("score") or {}).get("judge"), b.get("simulator") + + def sent_of(p: dict) -> Dict[str, Optional[str]]: + """Each model field's model as job ``p`` sends it; None where it has none (a control has no model of its own).""" + return {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, + "judge": (judge_model or providers.served_as(judge_def["model"], provider)) if judge_def else None, + "simulator": (sim_model or providers.served_as(sim_def["model"], provider)) if sim_def else None} + + def called_of(p: dict) -> List[str]: + s = sent_of(p) + return ledger.models(p["o"], s["model"], s["judge"], s["simulator"], None, control=p["control"]) + + # the pins by model, as the run sends each: a field's pin is its model's, on every job that calls that model + # (a control's judge job too, when the judge's model is the one the model field pinned) + by_model: Dict[str, List[str]] = {} + for p in picks: + for role, slugs in served.items(): + m = sent_of(p).get(role) + if not m: + continue + if m in by_model and by_model[m] != list(slugs): + add("model", f"Served by pins {m} twice, with different providers: give its fields the same ones.") + by_model[m] = list(slugs) + # each field's model as the run sends it, for the builder to ask its providers under that name + out["served_models"] = {role: next((sent_of(p)[role] for p in sorted(picks, key=lambda q: q["control"]) if sent_of(p)[role]), None) + for role in roles} + + def pinned(p: dict) -> Dict[str, List[str]]: + """The --served-by of one job: the pins of the models it calls.""" + called = called_of(p) + return {m: s for m, s in by_model.items() if m in called} + if served and not errors: + everything = {m for p in picks for m in called_of(p)} + for role in served: + if not {sent_of(p)[role] for p in picks} & everything: + add("model", f"Served by: no job of this recipe calls {roles[role]}; leave it empty.") if errors: return out nouns = (b.get("units") or {}).get("nouns", "units") @@ -535,7 +596,8 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--backend", "container"] if backend == "container" else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch] if of > 1 else []), - *limits.cli_words(limits.of(targets[index - 1])), "--from-run"] + *limits.cli_words(limits.of(targets[index - 1])), + *[w for m_, slugs in pinned(p).items() for w in ("--served-by", f"{m_}={','.join(slugs)}")], "--from-run"] return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"' def script_for(d: dict, p: dict, command: str, index: int) -> str: @@ -615,7 +677,8 @@ def row(p: dict) -> list: [STEP["model"], "Model", "none: a control calls no model" if out["keyless"] else f"{providers.label(providers.choice(provider, str(recipe.get('endpoint') or '')))} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only") + (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "") - + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")], + + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + + "".join(f" · {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ") + (" of each dataset" if many and isinstance(limit, int) else "") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a8c5945..76fa01d 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -9,10 +9,16 @@ byte for byte - and recorded once it is complete, one line per call in ``/.jsonl``: when it started and ended, the model, the status, the tokens, and an abridged request and reply. Three API shapes are read, -Chat Completions, Responses and Anthropic's Messages, streamed or not. The +Chat Completions, Responses and Anthropic's Messages, streamed or not. Read +apart from the rest (:func:`reply_facts`), the reply also says, when it does, +the serving provider that answered (``served_by``), how it ended (``end``) +and an error it holds though its status was a success (``error``); reading +them never changes what the caller receives, nor the rest of the record but a failed Responses object's cost. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the -user's price list, ``nullius.prices``), marked as an estimate. +user's price list, ``nullius.prices``), marked as an estimate. A Responses object answered 200 that failed (its ``end``) with no usage +costs 0, marked as failed: the provider keeps no generation for it, and a +key's usage compared with the relay's sum would show one billed after all. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -24,6 +30,12 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +A run may pin, for a model, the serving providers that may answer it +(``serving``, OpenRouter's routing): the relay then puts +``provider: {order, allow_fallbacks: false}`` on every request it sends as +that model, in place of the caller's own, and records both, cleaned of +credentials. Without a pin a request's body goes on as it did. + One answer is the relay's own: a run with a spending cap (``ledger``) books each call before it goes out, and a call that does not fit is not sent. Its caller gets a 402 that names the relay and the cap, and the record says @@ -31,6 +43,8 @@ """ from __future__ import annotations +import base64 +import binascii import hashlib import http.client import json @@ -67,6 +81,16 @@ # its next use, before any of the reply is read: worth one new connection. _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError) ANTHROPIC_VERSION = "2023-06-01" +ROUTE_REFUSED = (404, 503) +ROUTE_ERROR = "ApiProviderRouteError" +CONTEXT_ERROR = "ContextWindowExceededError" +STOP = "stop" +OVERFLOW = "context window" +COMPLETION = re.compile(r"^/(?:chat/completions|completions|messages|responses)$") +_WHOLE = ((b"\x89PNG\r\n\x1a\n", b"IEND\xaeB`\x82", "image/png"), (b"\xff\xd8\xff", b"\xff\xd9", "image/jpeg"), + (b"GIF87a", b";", "image/gif"), (b"GIF89a", b";", "image/gif")) +_PROVIDER = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._()/-]{0,79}") +_CONTEXT = re.compile(r"context[ _-]?(length|window|limit)|prompt is too long|token count .{0,40}exceeds", re.I) def _cut(text, n=4000): @@ -104,7 +128,7 @@ def _usage(u: Optional[dict]) -> Optional[dict]: inp += cached + written else: written = (det or {}).get("cache_write_tokens") or 0 - got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0} + got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or (rdet or {}).get("thinking_tokens") or 0} if written: got["cache_write"] = written hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0 @@ -138,6 +162,25 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def _hollow(path: str, body: bytes) -> Optional[tuple]: + """A 200 to a model call that holds no reply: the status it stands for, 429 or 402 when an error it held says so, else + 502, and its kind alone, since its words can quote the request.""" + if not COMPLETION.search(path.split("?")[0]): + return None + try: + got = json.loads(body) + except ValueError: + return 502, "no reply" + if not isinstance(got, dict): + return 502, "no reply" + if read_reply(path, body, False): + return None + code = got["error"].get("code") if isinstance(got.get("error"), dict) else None + code = code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code < 600 else None + return ((code if code in (402, 429) else 502), "an error" + (f" {code}" if code else "")) if got.get("error") \ + else (502, "no reply") + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: """What a reply said: its text, its tool calls, its usage.""" if not streamed: @@ -192,7 +235,7 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): t, c = _text_of_responses(ev.get("response") or {}) return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: @@ -211,6 +254,219 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: "usage": _usage(told) if told is not None else usage} +# -- serving providers: the pin a run sets, and what a reply says of who served it -- +SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def routing(slugs) -> dict: + """OpenRouter's routing field for a strict pin: these serving providers, + tried in this order, and no other.""" + return {"order": list(slugs), "allow_fallbacks": False} + + +def serving_problems(serving) -> List[str]: + """What is wrong with a pin by model (``{model: [slug, ...]}``): the one + check the run, its command line and the relay's own command share.""" + if not isinstance(serving, dict): + return ["the serving providers are given by model: {MODEL: [SLUG, ...]}"] + bad = [] + for model, slugs in serving.items(): + if not isinstance(model, str) or not model.strip(): + bad.append("a pin names no model") + continue + if not isinstance(slugs, (list, tuple)) or not slugs: + bad.append(f"{model}: no serving provider") + continue + wrong = [s for s in slugs if not isinstance(s, str) or not SLUG.fullmatch(s)] + if wrong: + bad.append(f"{model}: {', '.join(repr(s) for s in wrong)} is not a serving provider's slug " + "(as a model's page names it: deepinfra, deepinfra/fp8)") + elif len(set(slugs)) != len(slugs): + bad.append(f"{model}: a serving provider is named twice") + return bad + + +def parse_served_by(given, model: str) -> tuple: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable, as ``({model: [slug, ...]}, problems)``. Without + ``MODEL=`` it is ``model``, the run's. A model named twice is a problem; so is each of :func:`serving_problems`.""" + out: Dict[str, List[str]] = {} + bad = [] + for g in given or []: + name, eq, listed = str(g).rpartition("=") + name = name.strip() if eq else (model or "") + if not name: + bad.append(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") + continue + if name in out: + bad.append(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") + continue + out[name] = [s.strip() for s in listed.split(",")] + return out, bad + serving_problems(out) + + +def _clean(value, secrets=()): + """A recorded copy, never what is sent: a value under a credential's name + replaced, the relay's own key cut out of every string - a key of a + mapping as much as a value - then ``redact``.""" + if isinstance(value, dict): + return {_clean(k, secrets): "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) + for k, v in value.items()} + if isinstance(value, list): + return [_clean(v, secrets) for v in value] + if isinstance(value, str): + for s in secrets: + if s: + value = value.replace(s, "[REDACTED]") + return redact(value) + return value + + +def _name(v) -> Optional[str]: + return v if isinstance(v, str) and v else None + + +def _an_error(e) -> Optional[dict]: + """An error object, in one shape: its code (a word or a number), type and + message (words), as given; None when it has none of them of that kind. + Nothing else of the object is kept.""" + if not isinstance(e, dict) or not e: + return None + got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} + if isinstance(e.get("message"), str): + got["message"] = _cut(e["message"], 500) + return got or None + + +def _routing_facts(obj) -> Optional[list]: + """The serving endpoints OpenRouter's routing facts (``openrouter_metadata``) + list as available, in an object of a reply, its ``message`` or its + ``response``; None when none of them holds such a list.""" + if not isinstance(obj, dict): + return None + for o in (obj, obj.get("message"), obj.get("response")): + meta = o.get("openrouter_metadata") if isinstance(o, dict) else None + endpoints = meta.get("endpoints") if isinstance(meta, dict) else None + available = endpoints.get("available") if isinstance(endpoints, dict) else None + if isinstance(available, list): + return available + return None + + +def _provider(obj) -> Optional[str]: + """The serving provider an object of a reply names: its own ``provider``, + its ``message``'s or its ``response``'s, else the one endpoint OpenRouter's + routing facts mark ``selected`` among the available ones (asked for with + ``X-OpenRouter-Metadata``; a Responses reply names its provider there + alone). Facts that mark no single named endpoint give none.""" + if not isinstance(obj, dict): + return None + for o in [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)]: + if _name(o.get("provider")): + return o["provider"] + chosen = [a for a in _routing_facts(obj) or () if isinstance(a, dict) and a.get("selected") is True] + return _name(chosen[0].get("provider")) if len(chosen) == 1 else None + + +def _choices(choices, ends: dict, errors: dict) -> None: + """Each choice's end and error, by its index.""" + if not isinstance(choices, list): + return + for n, ch in enumerate(choices): + if not isinstance(ch, dict): + continue + i = ch.get("index") if isinstance(ch.get("index"), int) and not isinstance(ch.get("index"), bool) else n + if _name(ch.get("finish_reason")): + ends[i] = ch["finish_reason"] + e = _an_error(ch.get("error")) + if e and i not in errors: + errors[i] = e + + +def _status(resp) -> Optional[str]: + """A Responses object's end: its status, and why when it is incomplete.""" + if not isinstance(resp, dict) or not _name(resp.get("status")): + return None + why = resp.get("incomplete_details") + why = _name(why.get("reason")) if isinstance(why, dict) else None + return f"{resp['status']}: {why}" if why else resp["status"] + + +def _facts(path: str, body: bytes, streamed: bool) -> dict: + served, end, error, ends, errors = None, None, None, {}, {} + if not streamed: + got = json.loads(body) + if not isinstance(got, dict): + return {} + served, error = _provider(got), _an_error(got.get("error")) + _choices(got.get("choices"), ends, errors) + if got.get("type") == "message": + end = _name(got.get("stop_reason")) + elif got.get("object") == "response" or "output" in got: + end = _status(got) + else: + for line in body.decode("utf-8", "replace").splitlines(): + if not line.startswith("data:"): + continue + try: + ev = json.loads(line[5:].strip()) + except ValueError: + continue + if not isinstance(ev, dict): + continue + # the latest event that names one; routing facts that mark no single named endpoint leave none + served = _provider(ev) if _routing_facts(ev) is not None else (_provider(ev) or served) + kind = ev.get("type") + if kind == "error": # Messages' error event, or a Responses one + error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) + elif isinstance(ev.get("error"), dict): # an error chunk under a 200 + error = error or _an_error(ev["error"]) + if kind == "message_delta" and isinstance(ev.get("delta"), dict): + end = _name(ev["delta"].get("stop_reason")) or end + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): + resp = ev.get("response") + end = _status(resp) or end + error = error or _an_error(resp.get("error") if isinstance(resp, dict) else None) + _choices(ev.get("choices"), ends, errors) + if ends: + end = ends[min(ends)] if len(ends) == 1 else {str(i): ends[i] for i in sorted(ends)} + if error is None and errors: + i = min(errors) + error = dict(errors[i], choice=i) + return {k: v for k, v in (("served_by", served), ("end", end), ("error", error)) if v is not None} + + +def reply_facts(path: str, body: bytes, streamed: bool) -> dict: + """What a reply says besides what :func:`read_reply` reads, read apart from + it: the serving provider that answered (``served_by``), how it ended + (``end``: ``finish_reason``, ``stop_reason`` or a Responses status; one per + choice, by index, when there are several) and an error it holds though its + status was a success (``error``: code, type and message; a top-level one + whatever else the reply holds, else a choice's). A fact of the wrong type + is left out, and a reply that cannot be read gives none: nothing here can + stop a reply reaching its caller.""" + try: + return _facts(path, body, streamed) + except Exception: + return {} + + +def _plain(name) -> str: + """A provider's name or slug, compared: lower case, letters and digits.""" + return re.sub(r"[^a-z0-9]", "", str(name).split("/")[0].lower()) + + +def held(provider: str, only: List[str]) -> Optional[bool]: + """True when the provider that answered is one of ``only``, False when it shares nothing with them, None when + it cannot be told: a name is not always its slug.""" + name = _plain(provider) + slugs = {_plain(p) for p in only} + if not name or name in slugs: + return True if name else None + if any(s.startswith(name) or name.startswith(s) for s in slugs if s): + return None + return False + + def _request_tail(req: dict) -> dict: """The part of a request a reader needs: the model, how many messages, the last two, abridged, and the reasoning effort it asked for (chat's @@ -236,6 +492,37 @@ def _request_tail(req: dict) -> dict: "reasoning_effort": effort if isinstance(effort, str) else None} +def _image(data: str) -> Optional[str]: + """An inline image's type when its bytes show a whole one: its head, and its end after it.""" + try: + raw = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + return None + if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP": + return "image/webp" if int.from_bytes(raw[4:8], "little") + 8 <= len(raw) else None + if raw[:2] == b"BM": + return "image/bmp" if int.from_bytes(raw[2:6], "little") <= len(raw) else None + return next((t for head, tail, t in _WHOLE if raw.startswith(head) and tail in raw[len(head):]), None) + + +def _images(x) -> List[dict]: + """Each inline image a request carries, as Messages or chat sends one: its declared type, the type its bytes + show whole (None when they show none) and its size, never a byte of it.""" + if isinstance(x, list): + return [i for v in x for i in _images(v)] + if not isinstance(x, dict): + return [] + src, url = x.get("source"), x.get("image_url") + url = url.get("url") if isinstance(url, dict) else url + if x.get("type") == "image" and isinstance(src, dict) and src.get("type") == "base64" and isinstance(src.get("data"), str): + said, data = src.get("media_type"), src["data"] + elif isinstance(url, str) and url.startswith("data:") and ";base64," in url: + said, data = url[5:].split(";", 1)[0], url.split(";base64,", 1)[1] + else: + return _images(list(x.values())) + return [{"type": str(said)[:40] if said else None, "is": _image(data), "bytes": len(data)}] + + def _digest(obj) -> str: text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True) return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16] @@ -292,8 +579,9 @@ class _Pool: KEEP = 16 # idle connections kept per provider; more are closed - def __init__(self, base: str): + def __init__(self, base: str, timeout: float = TIMEOUT): u = urllib.parse.urlsplit(base) + self.timeout = timeout self.https = u.scheme == "https" self.host, self.port = u.hostname, u.port or (443 if self.https else 80) self.prefix = u.path.rstrip("/") @@ -309,7 +597,7 @@ def take(self, fresh: bool = False): kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection with self.lock: self.opened += 1 - return kind(self.host, self.port, timeout=TIMEOUT), False + return kind(self.host, self.port, timeout=self.timeout), False def give(self, conn, r): """Back to the pool when the reply was read whole and the provider @@ -355,7 +643,8 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, - ledger: Optional[str] = None, unit_caps: bool = True): + ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, + facts: Optional[bool] = None, timeout: float = TIMEOUT): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -373,8 +662,16 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s (:func:`request_fields`) - ``fidelity``'s, never a normal run's. ``ledger``: the file of the run's spending cap (``ledger``), shared by every relay of the job: a call is booked there before it is sent, - and not sent when it does not fit.""" + and not sent when it does not fit. ``serving``: by model, the serving + providers that may answer a request sent as that model (:func:`routing`), + put on every such request in place of the caller's own. ``facts``: a + pinned Chat Completions or Responses request asks OpenRouter for its + routing facts, where its reply names who served it; by default when + ``base`` is OpenRouter's. ``timeout``: the seconds one call may wait.""" self.fields = fields + self.serving = {m: list(s) for m, s in (serving or {}).items()} + host = (urllib.parse.urlsplit(base).hostname or "").lower() + self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) self.ledger = ledgers.Ledger(ledger) if ledger else None # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them self.unit_caps = unit_caps @@ -386,7 +683,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s self.models: Dict[str, str] = {} self.embedding_models: Dict[str, str] = {} self.lock = threading.Lock() - self.pool = _Pool(self.base) + self.pool = _Pool(self.base, timeout) self.server: Optional[ThreadingHTTPServer] = None os.makedirs(record_dir, exist_ok=True) @@ -437,8 +734,14 @@ def _said(self, e: Exception) -> str: text = text.replace(k, "[REDACTED]") return redact(text)[:200] - def _headers(self, caller) -> Dict[str, str]: + def _headers(self, caller, facts: bool = False) -> Dict[str, str]: + """``facts``: a pinned Chat Completions or Responses request to + OpenRouter, sent asking for its routing facts whatever the caller + said of them.""" out = forwarded(caller) + if facts: + out = {k: v for k, v in out.items() if k.lower() != "x-openrouter-metadata"} + out["X-OpenRouter-Metadata"] = "enabled" has = {k.lower() for k in out} # a name in any case is the same header: never send it twice for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"): @@ -450,8 +753,13 @@ def _headers(self, caller) -> Dict[str, str]: out["Authorization"] = "Bearer " + self.key return out + def _asked(self) -> bool: + return asked(self.dir) is not None + def _forward(self, h: BaseHTTPRequestHandler): - """One call of a caller, ``h``: sent on, passed back, recorded.""" + """One call of a caller, ``h``: sent on, passed back, recorded. A caller's own call + refused for its context window also leaves STOP beside the records; one made once + another asked by STOP is refused unsent and unrecorded.""" bits = h.path.split("/", 2) if len(bits) < 3 or not _TAG.match(bits[1]): h.send_error(404, "no route: //v1/...") @@ -465,6 +773,7 @@ def _forward(self, h: BaseHTTPRequestHandler): req = {} asked = req.get("model") if isinstance(req, dict) else None side = False + routing_asked, pinned = None, None if isinstance(req, dict) and req: if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag): req["model"] = self.embedding_models[tag] @@ -477,7 +786,14 @@ def _forward(self, h: BaseHTTPRequestHandler): req["model"] = self.model if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"): req.setdefault("stream_options", {})["include_usage"] = True + pinned = self.serving.get(req["model"]) if isinstance(req.get("model"), str) else None + if pinned: + routing_asked = req.get("provider") + req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if not side and not self.models.get(tag) and self._asked(): + h.send_error(400, "the unit was asked to stop") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -498,8 +814,11 @@ def _forward(self, h: BaseHTTPRequestHandler): t0 = time.time() conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed + cut_off = None try: - conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers)) + # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself + facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) + conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: status, kind = 502, "application/json" @@ -516,28 +835,55 @@ def _forward(self, h: BaseHTTPRequestHandler): except Exception as e: # cut off mid-reply status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() + empty = _hollow(rest, body) if status == 200 and not streamed and h.command == "POST" \ + and not (isinstance(req, dict) and req.get("stream")) else None + if empty: + (status, inside), kind = empty, "application/json" + back = [(n, v) for n, v in back if n.lower() in ("retry-after", "retry-after-ms")] + body = json.dumps({"error": {"message": f"the endpoint answered 200 with {inside}"}}).encode() def record(body): rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None, - "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {}, + "model_asked": asked, "status": status, + "request": dict(_request_tail(req), sha256=_digest(req)) if isinstance(req, dict) else {}, "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True + if isinstance(req, dict) and req.get("provider") is not None: + # the routing as sent, and the caller's own the pin replaced: recorded copies, cleaned + rec["request"]["routing"] = _clean(req["provider"], (self.key, self.key.strip())) + if pinned and routing_asked is not None: + rec["request"]["routing_asked"] = _clean(routing_asked, (self.key, self.key.strip())) if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None - if status == 200: + if status == 200 or cut_off: rec["reply"] = read_reply(rest, body, streamed) - cost, how = self._cost(rec["model"], rec["reply"].get("usage")) + # who served it, how it ended, an error under the 200: beside what was read, never in its place + try: + rec["reply"].update({k: v for k, v in _clean(reply_facts(rest, body, streamed), (self.key, self.key.strip())).items() + if k not in rec["reply"]}) + except Exception: + pass + cost, how = (0.0, "failed") if rec["reply"].get("end") == "failed" and not rec["reply"].get("usage") \ + else self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - else: - rec["error"] = _said(status, body) + if status != 200: + rec["error"] = cut_off or _said(status, body) + sent = _images(req) if status in (400, 413, 422) else [] + if sent: + rec["images"] = sent + if empty: + rec["answered"] = 200 if booking is not None: rec["booked_usd"] = booked try: - if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) + if status == 200 or cut or empty: # what it cost; unknown (no token count, cut off), the booking is kept whole + whole = not (cut_off or rec.get("cost_from") == "failed") + known = cost if whole or (cost is not None and cost > booked) else None + self.ledger.settle(booking, known, rec["model"], + ((rec.get("reply") or {}).get("usage") or {}).get("output") if whole else None) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -545,6 +891,9 @@ def record(body): with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") + if not side and not self.models.get(tag) and overflowed(rec): + with open(os.path.join(self.dir, STOP), "a") as f: + f.write(OVERFLOW + "\n") h.send_response(status) if not any(n.lower() == "content-type" for n, _ in back): @@ -560,6 +909,8 @@ def record(body): while True: chunk = r.read1(65536) if not chunk: + if r.length: + raise http.client.IncompleteRead(b"", r.length) whole = True break parts.append(chunk) @@ -568,8 +919,8 @@ def record(body): h.wfile.flush() except OSError: pass # the caller went away; the record still completes - except Exception: - pass + except Exception as e: + status, cut, cut_off = 502, True, "the endpoint's reply was cut off: " + self._said(e) record(b"".join(parts)) # before the connection closes: the caller's end of stream else: record(body) # before the caller has the reply, so a reader after it finds the call @@ -693,11 +1044,26 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True -def failure(calls: List[dict]) -> Optional[tuple]: +def asked(record_dir: str) -> Optional[str]: + """What a guard's STOP in ``record_dir`` says, or None: none there, or only the context window's.""" + try: + with open(os.path.join(record_dir, STOP)) as f: + said = next((ln.strip() for ln in f.read().splitlines() or [""] if ln.strip() != OVERFLOW), None) + except OSError: + return None + return None if said is None else said or STOP + + +def failure(calls: List[dict], words: bool = True, agent: bool = False, serving: Optional[Dict[str, List[str]]] = None) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's). A call + succeeded (a failure the agent recovered from is not the unit's). + ``words`` False: the status alone, never the provider's text, which can quote + a gated task. ``agent``: the agent's last call over the model's context + window is ``ContextWindowExceededError``, its own, whatever side call followed, and a side call's 200 does not + recover the agent's own. A pinned call (``serving``) no endpoint took is + ``ApiProviderRouteError``. Calls are taken in the order they began. A call the run's spending cap refused is the unit's wherever it falls: once one is refused, every later one is.""" cut = next((c for c in calls if c.get("refused") == "cap"), None) @@ -706,26 +1072,81 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or calls[-1].get("status") == 200: + calls = _begun(calls) + ok = [i for i, c in enumerate(calls) if c.get("status") == 200] + after, side = max((i for i in ok if not (agent and calls[i].get("side"))), default=-1), max(ok, default=-1) + calls = [c for i, c in enumerate(calls) if i > after and c.get("status") != 200 and not (c.get("side") and i < side)] + if not calls: return None + own = next((c for c in reversed(calls) if not c.get("side")), None) if agent else None + if own is not None and overflowed(own): + said = (own.get("error") or "") if words else ("the endpoint answered 400 to the agent's call, longer than the model's " + "context window; its words are not kept, the benchmark's tasks being gated") + return CONTEXT_ERROR, said last = calls[-1] - s, said = last.get("status"), last.get("error") or "" + s = last.get("status") + said = (last.get("error") or "") if words else f"the endpoint answered {s}; its words are not kept, the benchmark's tasks being gated" + if (serving or {}).get(last.get("model")) and s in ROUTE_REFUSED: + return ROUTE_ERROR, _refusal(last, said, serving) if s == 429: return "RateLimitError", said if s in (401, 403): return "AgentAuthenticationError", said - if s == 404 and "model" in said.lower(): + if s == 404 and "model" in (last.get("error") or "").lower(): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said return "ApiError", said +def _begun(calls: List[dict]) -> List[dict]: + timed = all(isinstance(c.get("t0"), (int, float)) for c in calls) + return sorted(calls, key=lambda c: c["t0"]) if timed else list(calls) + + +def overflowed(call: dict) -> bool: + """A 400 for a request longer than the model's context window, by the provider's words.""" + return call.get("status") == 400 and bool(_CONTEXT.search(call.get("error") or "")) + + +def _refusal(call: dict, said: str, serving: Dict[str, List[str]]) -> str: + """A pinned route's refusal: its status first (route_dead reads it), the model, its pin, then ``said``.""" + text = f"{call.get('status')}: no endpoint of {call.get('model')} pinned to {', '.join(serving[call.get('model')])} took the call" + return f"{text}: {said}" if said else text + + +def routes_failure(routes: List[List[dict]], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """:func:`route_failure` of each route's calls on its own, the first found.""" + return next(filter(None, (route_failure(calls, serving, words) for calls in routes)), None) + + +def route_dead(exc: Optional[tuple]) -> bool: + """A pinned route with no endpoint at all (404): every later unit would fail the same way.""" + return bool(exc) and exc[0] == ROUTE_ERROR and str(exc[1]).startswith("404: ") + + +def route_failure(calls: List[dict], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """A pinned route (``serving``) that failed a unit, among one route's ``calls``: a model whose last call was + refused, or a call its reply says another provider served, by a name that reads as one.""" + last = {} + for c in _begun(calls): + last[c.get("model")] = c + by = (c.get("reply") or {}).get("served_by") + if serving.get(c.get("model")) and isinstance(by, str) and _PROVIDER.fullmatch(by) and held(by, serving[c.get("model")]) is False: + return ROUTE_ERROR, f"{c.get('status')}: {c.get('model')} pinned to {', '.join(serving[c.get('model')])} was served by {by}" + for c in last.values(): + if serving.get(c.get("model")) and c.get("status") in ROUTE_REFUSED: + said = (c.get("error") or "") if words else \ + f"the endpoint answered {c.get('status')}; its words are not kept, the benchmark's tasks being gated" + return ROUTE_ERROR, _refusal(c, said, serving) + return None + + # -- the relay as a process of its own: a container backend's sidecar ---------- def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME] - [--public URL] [--prices JSON] [--ledger FILE]``. + [--public URL] [--prices JSON] [--ledger FILE] [--served-by MODEL=SLUG[,SLUG...] ...]``. ``--base`` is the provider's endpoint, ``--route`` a route sent as another model (a benchmark's judge), ``--public`` the address ``url`` hands out. @@ -747,7 +1168,26 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", + help="the serving providers that may answer a request sent as MODEL, in order, and no other; " + "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " + "or --routing given") + ap.add_argument("--routing", action="store_true", + help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") + ap.add_argument("--timeout", type=float, default=TIMEOUT, help="seconds one call may wait for the provider's reply") a = ap.parse_args(argv) + serving, bad = parse_served_by(a.served_by, "") + if serving and not bad: + host = (urllib.parse.urlsplit(a.base).hostname or "").lower() + if not (a.routing or host == "openrouter.ai" or host.endswith(".openrouter.ai")): + bad.append(f"{a.base} routes no model to several serving providers (OpenRouter's does; --routing says another does)") + sent = {a.model} | {r.partition("=")[2] for r in a.route} | {(s.partition("=")[2] or s) for s in a.side_model} + stray = [m for m in serving if a.model and m not in sent] + if stray: + bad.append(f"{', '.join(stray)}: a model this relay does not send (it sends {', '.join(sorted(x for x in sent if x))})") + if bad: + print(json.dumps({"error": "--served-by: " + "; ".join(bad)}), flush=True) + return 2 try: with open(a.key_file) as f: key = f.read().strip() @@ -759,7 +1199,8 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, + timeout=a.timeout) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 236c8b6..ce5854a 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -51,6 +51,7 @@ """ from __future__ import annotations +import fcntl import hashlib import json import os @@ -66,7 +67,7 @@ from collections import Counter from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from contextlib import contextmanager -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact @@ -100,7 +101,8 @@ def driver_path() -> str: PROVIDERS = { "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, - "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"}, + # routing: one model is served by several providers, and a request may say which may answer (--served-by) + "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY", "routing": True}, "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } @@ -113,6 +115,10 @@ def driver_path() -> str: _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) +HELD: dict = {} +LIVE: dict = {} + + class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" @@ -161,6 +167,19 @@ def units_file(path: str) -> dict: return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} +def served_by(given, model: str) -> Dict[str, List[str]]: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable: by model, the + serving providers that may answer it, in the order they are tried, and no + other (OpenRouter's slugs, as a model's page gives them: ``deepinfra``, + ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused as + ``relay.parse_served_by`` finds it wrong: the check the run and the + relay's own command make too.""" + out, bad = relay.parse_served_by(given, model) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + return out + + def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit" @@ -346,7 +365,37 @@ def get(self, name: str) -> Optional[dict]: return {"kind": kind, "value": text} if text else None +def resolve_bench_options(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + which=shutil.which) -> Tuple[dict, dict]: + """The benchmark's options for a run on ``dataset`` and what their values need; refused before anything starts.""" + vals, bad = definitions.bench_options_of(bench, dataset, given, setup) + if bad: + raise Refused("; ".join(bad)) + needs = definitions.option_needs(bench, vals) + lacking = hostreq.problems(bench, needs["host"], which) + if lacking: + raise Refused(f"{', '.join(f'{k}={v}' for k, v in vals.items())} needs on this machine " + + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) + else ". Run it on a machine that has it")) + return vals, needs + + +def pins_of(bench: dict, other: dict, provider: str, judge_model: Optional[str]) -> Dict[str, List[str]]: + """{model as sent: [provider slug]}: the original setup's pins and the official judge's, on a provider that pins.""" + if provider not in definitions.PIN_PROVIDERS: + return {} + pins = {m: list(o) for m, o in (other.get("pins") or {}).items()} if other.get("_kind") == "upstream" else {} + j = (bench.get("score") or {}).get("judge") or {} + only = (j.get("pin") or {}).get(provider) + if only and judge_model and judge_model == providers.served_as(j["model"], provider): + pins[judge_model] = list(only) + return pins + + class Run: + call_seconds: float = relay.TIMEOUT + def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -356,7 +405,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -402,7 +452,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas agents on the local backend). Before the first unit the container constraints are checked on the engine (where those containers run), and the scope's on systemd: on the local backend a cap no scope holds is refused, on the container - backend its drivers then run without one.""" + backend its drivers then run without one. + ``served_by``: by model, the serving providers that may answer it + (:func:`served_by`), put on every request sent as that model by the + run's relays (``relay.routing``) and recorded in the job's config. + Refused when the provider routes no model to several (only + ``openrouter`` does), or when it names a model the run does not call.""" backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") @@ -526,6 +581,24 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) + self.serving = dict(served_by or {}) + if self.serving: + bad = relay.serving_problems(self.serving) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + self.serving = {m: list(s) for m, s in self.serving.items()} + if not PROVIDERS.get(provider, {}).get("routing"): + raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " + "among; the option needs a provider that routes a model to several (openrouter)") + # a control calls no model of its own and plays no user: only its judge and an option's models + called = ([m for m in (self.judge_model, self.embed_model, self.rerank_model) if m] if other.get("_kind") == "control" + else self.models_called()) + stray = [m for m in self.serving if m not in called] + if stray: + raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + + (", ".join(called) or "no model")) + self.route_pins = pins_of(bench, other, provider, self.judge_model) + self.serving.update(self.route_pins) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -541,6 +614,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + self.score_seconds = (bench.get("limits") or {}).get("score_seconds") or DRIVER_SECONDS + self.call_seconds = (bench.get("limits") or {}).get("call_seconds") or relay.TIMEOUT self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -608,8 +683,25 @@ def driver(self) -> jsonl.Proc: return d def start(self) -> str: + lck, mine = os.path.join(self.job_dir, ".work", "job.lck"), object() + try: + return self._start(lck, mine) + finally: + if LIVE.get(lck) is mine: + del LIVE[lck] + + def _start(self, lck, mine) -> str: t0 = time.time() os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True) + held = HELD.get(lck) or open(lck, "a") + try: + if LIVE.setdefault(lck, mine) is not mine: + raise BlockingIOError + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused(f"another run under the job name {self.name} is running: " + "start this one once it ends, or under another name") from None + HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) @@ -682,7 +774,7 @@ def start(self) -> str: # a shard of a list splits the listed units: they are its whole shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, - repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, + repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -755,7 +847,8 @@ def work_unit(iur): self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) self.capped = bool(budget and budget["stopped"]) - job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget) + job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget, + served=job.served(self.benchmark_calls(), getattr(self, "serving", None))) errored = sum(1 for x in results if x.get("exception_info")) self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results)) # the units' stage says why it ended early: a line of its own would be a stage that never starts @@ -807,11 +900,14 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, job's ``retried/`` (kept whole, its calls and its cost with it, where no reader takes it for a trial of its own), its second written under the trial's own name and counted (into ``done``, as each ends). Nothing - is run again once the run is stopped or halted; the spending cap goes - on counting every call.""" + is run again once the run is stopped or halted, nor a unit a guard's + stop asked to stop; the spending cap goes on counting every call.""" + if (self.bench.get("limits") or {}).get("run_again") is False: + return by_name = {self.trial_name(*iur): iur for iur in work} todo = [by_name[r["trial_name"]] for r in results - if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name] + if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name + and relay.asked(os.path.join(self.job_dir, r["trial_name"], ".work", "relay")) is None] if not todo: return self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time") @@ -917,6 +1013,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False, + serving=self.serving, timeout=self.call_seconds, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -955,6 +1052,11 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home") + earlier = os.path.join(trial, ".work", f"earlier.{time.time_ns()}") + for part in (os.path.join(".work", "relay"), "result.json", "verifier", "agent"): + if os.path.lexists(os.path.join(trial, part)): + os.makedirs(earlier, exist_ok=True) + os.replace(os.path.join(trial, part), os.path.join(earlier, os.path.basename(part))) os.makedirs(ws, exist_ok=True) # The workspace is a repository of its own: agents look up to the git root # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept @@ -1027,17 +1129,17 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.score_seconds + (float(step.get("seconds") or self.turn_seconds) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.score_seconds, **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e)) + raise _Ended(_kind(e, "VerifierError"), str(e)) except _Ended as e: exc = (e.kind, e.said) traj = traj or e.trajectory @@ -1056,17 +1158,27 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: for ln in getattr(rl, "mcp_listeners", None) or []: ln.stop() calls = rl.calls(tag) if rl is not None else [] - failed = relay.failure(calls) + words = not self.bench.get("gated") + failed = relay.failure(calls, words, agent=True, serving=getattr(self, "route_pins", None)) if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is # the endpoint's, and a cap's refusal is the cap's failed = None - if not failed and rl is not None and exc is not None and exc[0] == "VerifierError": + if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") + or self.upstream and exc and ": VerifierError: " in exc[1]): + failed = None + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and (score or {}).get("stopped") not in (None, "", relay.OVERFLOW) \ + and rl is not None and relay.asked(rl.dir) is not None \ + and [c.get("status") for c in relay._begun(calls) if not c.get("side")][-1:] == [429]: + failed = None + if not failed and rl is not None and getattr(self, "route_pins", None): + failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) + if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): # the judge's or the simulated user's model failing (a 429, a refused request) is the # unit's infrastructure, as the agent's own would be - not a miss counted against it - failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) + failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}"), words) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score = None @@ -1093,7 +1205,10 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), - environment=environment or "", + environment=environment or "", serving=getattr(self, "serving", None), + # who served the unit: its agent's calls and its judge's and simulated user's routes + served_calls=calls + ([c for side in ("judge", "simulator") for c in rl.calls(f"{tag}-{side}")] + if rl is not None else []), image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" @@ -1164,7 +1279,7 @@ def note_refusal(self, exc: Optional[tuple]) -> None: if getattr(self, "_halt", None) is None: return with self._said: - if exc and exc[0] in REFUSALS: + if exc and (exc[0] in REFUSALS or relay.route_dead(exc)): self._refused.append(exc) if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None: self.halted = exc @@ -1278,6 +1393,16 @@ def ledger_file(self) -> Optional[str]: led = getattr(self, "ledger", None) return led.path if led is not None else None + def benchmark_calls(self) -> List[dict]: + """The calls of the benchmark's own relay (an option's embeddings or reranking), every route of it.""" + folder = os.path.join(self.job_dir, ".work", "relay-benchmark") + calls = [] + for name in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + if name.endswith(".jsonl"): + with open(os.path.join(folder, name)) as f: + calls += [json.loads(line) for line in f if line.strip()] + return calls + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the @@ -1302,7 +1427,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file()).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1331,7 +1457,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds if self.call_seconds != relay.TIMEOUT else None, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1850,8 +1977,8 @@ def send(op, **kw): def drive_upstream(self, unit, ws, tag, transcript, rl): """The setup's unit: the driver runs the upstream's own answerer on it (``answer``), asked as ``asked`` through the unit's relay, which sends - the run's model in its place. A reply with no text - a reasoning model - that spent the upstream's token budget thinking - is said as such.""" + the run's model in its place. A last reply with no text or tool call - a reasoning model + that spent the upstream's token budget thinking - is said as such; a unit a guard stopped is cancelled.""" o = self.other try: # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes @@ -1860,7 +1987,18 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) except jsonl.Failed as e: calls = rl.calls(tag) - silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] + stop = relay.asked(rl.dir) + if stop and not _kind(e, None): + raise _Ended(CANCELLED, f"a guard asked the unit to stop ({stop[:200]}); {o['label']}: {e}", + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + own = [c for c in relay._begun(calls) if not c.get("side")][-1:] + failed = [((c["reply"].get("error") or {}).get("code") or "no code") for c in own if (c.get("reply") or {}).get("end") == "failed"] + kind = _kind(e, "ApiError" if failed else None) + if kind: + raise _Ended(kind, f"{o['label']}: {e}" + "".join(f"; {self.model}'s last reply failed ({c})" for c in failed), + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + silent = [c for c in own if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip() + and not (c.get("reply") or {}).get("tool_calls")] if silent: u = (silent[-1].get("reply") or {}).get("usage") or {} thought = u.get("reasoning") or 0 @@ -1875,6 +2013,12 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") +def _kind(e, otherwise): + """A driver's own EnvironmentSetupError, VerifierError, CancelledError or CauseUnknownError keeps its kind.""" + m = re.match(r"\w+: (EnvironmentSetupError|VerifierError|CancelledError|CauseUnknownError): ", str(e)) + return m.group(1) if m else otherwise + + class _Ended(Exception): def __init__(self, kind, said, trajectory=None): super().__init__(said) diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index d20fd5b..395952f 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -69,6 +69,7 @@ r.model_choice = choice; if (paperListed(r)) r.model = ''; if (choice !== 'any') { delete r.judge_model; delete r.simulator_model; } + if (choice !== was) delete r.served_by; // its models change with it: a pin belongs to the model it was set for if (lb && was === 'leaderboard' && choice !== 'leaderboard') { r.bench_options = {...(r.bench_options || {})}; for (const k of Object.keys(lb.bench_options || {})) delete r.bench_options[k]; @@ -79,6 +80,34 @@ } return r; } + // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are + // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. + const SERVED_ROLES = ['model', 'simulator', 'judge']; + function servedRoles(r) { + const b = standardBench(r); + if (!b || r.provider !== 'openrouter' || standardKeyless(r)) return []; + return SERVED_ROLES.filter(k => k === 'model' || (k === 'simulator' && b.simulator) || (k === 'judge' && b.judge)); + } + // A field's text, "deepinfra, together", as the pin's list, every part kept for the server to check ("a,,b" is + // refused in sight, as the option refuses it); a field with nothing in it pins nothing. + const servedList = text => { const parts = String(text || '').split(',').map(s => s.trim()); return parts.some(Boolean) ? parts : []; }; + // A model field's role, and its model changed: the pin set for the model before it goes with it. + const SERVED_FIELD = {model: 'model', simulator_model: 'simulator', judge_model: 'judge'}; + function setModelOf(r, field, value) { + if (SERVED_FIELD[field] && r[field] !== value && r.served_by) setServed(r, SERVED_FIELD[field], []); + r[field] = value; + return r; + } + // The model a field's Served by lists the providers of: the one the run sends, as the server compiled it + // (recipe.py, served_models: a paper model's served name, an official one's with its maker), else the one shown. + const servedModelOf = (role, shown, compiled) => ((compiled || {}).served_models || {})[role] || shown || ''; + function setServed(r, role, slugs) { + const list = Array.isArray(slugs) ? slugs : servedList(slugs); + r.served_by = {...(r.served_by || {})}; + if (list.length) r.served_by[role] = list; else delete r.served_by[role]; + if (!Object.keys(r.served_by).length) delete r.served_by; + return r; + } function createStandard(benchmark = '', other = '') { const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0]; const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [], @@ -94,7 +123,7 @@ const b = standardBench({benchmark: key}); if (!b) return r; const was = standardBench(r); - if (was !== b) r.model_choice = 'any'; + if (was !== b) { r.model_choice = 'any'; delete r.served_by; } r.benchmark = key; const kept = was === b ? datasetsOf(r).filter(id => b.datasets.some(d => d.id === id)) : []; r.datasets = kept.length ? kept : [b.datasets[0].id]; @@ -141,6 +170,11 @@ // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); + // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) + if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) + for (const role of SERVED_ROLES) { const v = raw.served_by[role]; + // kept as saved, a slug the server refuses included: a pin is refused in sight, never dropped in silence + if (Array.isArray(v) && v.length) setServed(r, role, v.map(s => typeof s === 'string' ? s.trim() : String(s))); } r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; else r.model = bareModel(r.model); @@ -217,7 +251,8 @@ const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel); const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r); // The Model step's choice into the recipe: its provider, and for a named endpoint its id. - function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); } + // a provider that routes no model to several takes no Served by: a pin made for OpenRouter is dropped with it + function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); if (r.provider !== 'openrouter') delete r.served_by; } // Why Run cannot start this recipe here, or '': its provider is not one of those. A recipe naming one keeps it, // said, until another is chosen - never switched behind the person's back. The server refuses the same // (web/execute.py, provider_problem). @@ -759,5 +794,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,setModelOf,servedModelOf,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index e37e016..af45dec 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -246,3 +246,8 @@ #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +/* Served by: under its model field, set in from it: the typed list, then the providers to add from */ +#nullius-builder .std-served { margin: 10px 0 22px 18px; padding: 2px 0 2px 14px; border-left: 2px solid var(--nb-line); font-size: 14px; gap: 8px; } +#nullius-builder .std-served label { display: grid; gap: 6px; } +#nullius-builder .std-served select { max-width: 420px; } +@media(max-width:520px) { #nullius-builder .std-served { margin-left: 6px; padding-left: 10px; } } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index f4eb338..ff3793e 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -114,7 +114,9 @@ }; // The standard's recipe as the server compiles it from the definitions (POST // /new-run/runner). Nothing typed here is a command; Run sends the choices. - let runnerOut = null, runnerKey = '', runnerTimer; + let runnerOut = null, runnerKey = '', runnerTimer, runnerOutKey = ''; + // the server's compile of the recipe as it is now, or null while a newer one is asked + const compiledNow = () => runnerOutKey === JSON.stringify(r()) ? runnerOut : null; // Switching runner keeps the recipe being edited, one per runner, for as long as the page is open: switching back finds it as it was. const keptByRunner = {}; // Nullius's own mark for the standard; Harbor's for Harbor @@ -602,12 +604,13 @@ return `

Choose the model.

` + modelChoiceCards(rec) + providerField(rec, C.PROVIDERS) + field('Model', 'model', rec.model, {list:'nb-models', placeholder:paper && C.paperListed(rec) ? 'Choose a paper model' : 'Search or enter a model ID'}) + + servedField(rec, 'model', rec.model) // the benchmark's other models: the user simulator its conversations need, the judge that grades - each its official one // unless named; like the paper, the official one; like the leaderboard, its own and the options it fixes + (b.simulator ? (paper ? fixed('Model for the user simulator', lb && lb.simulator || b.simulator, lb && lb.simulator ? 'leaderboard' : 'official') : field('Model for the user simulator', 'simulator_model', rec.simulator_model || '', {list: 'nb-models', placeholder: b.simulator + ' (official)', - hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) : '') + hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) + servedField(rec, 'simulator', (!paper && rec.simulator_model) || lb && lb.simulator || b.simulator) : '') + (b.judge ? (paper ? fixed('Model for the judge', lb && lb.judge || b.judge, lb && lb.judge ? 'leaderboard' : 'official') : field('Model for the judge', 'judge_model', rec.judge_model || '', {list: 'nb-models', placeholder: b.judge + ' (official)', - hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) : '') + hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) + servedField(rec, 'judge', (!paper && rec.judge_model) || lb && lb.judge || b.judge) : '') + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, v, 'leaderboard')).join('') : '') + keyLine(rec); } @@ -637,7 +640,8 @@ out = await res.json(); } catch (e) { out = {errors: [{step: REVIEW, message: 'The recipe could not be compiled here: ' + e}], notes: [], review: []}; } if (key !== runnerKey) return; - runnerOut = out; + runnerOut = out; runnerOutKey = key; + if (step === S.model) fillServed(); if (step === REVIEW) renderForm(); refreshPlan(); }, 120); @@ -646,6 +650,41 @@ const select = (label,key,value,options) => ``; const fixed = (label,value,whose='official') => ``; const note = (title,text) => `
${title}${text}
`; + // Served by: under a model field, on OpenRouter alone, the serving providers that may answer it, in order (recipe.py, + // served_by); typed as slugs, or added from the model's providers (POST /new-run/endpoints) + const SERVED_NAME = {model: 'model', simulator: 'user simulator', judge: 'judge'}; + // its list: the providers of the model the run sends (the server's served_models), not of the name shown + const servedField = (rec, role, model) => !C.servedRoles(rec).includes(role) ? '' : + `
` + + `
`; + const servedCache = new Map(); + function servedList(model) { + let entry = servedCache.get(model); + if (!entry || Date.now() - entry.at > 300000) { + entry = {at: Date.now(), promise: nulliusFetch('/new-run/endpoints', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': schema.csrf}, body: JSON.stringify({provider: 'openrouter', model})}) + .then(x => x.json()).then(x => ({...x, endpoints: Array.isArray(x.endpoints) ? x.endpoints : []})) + .catch(() => ({ok: false, endpoints: [], message: "Provider list unavailable. Enter a provider's slug by hand."}))}; + servedCache.set(model, entry); + } + return entry.promise.then(x => { if (!x.ok) servedCache.delete(model); return x; }); + } + function fillServed() { + root.querySelectorAll('select[data-served-add]').forEach(async select => { + // the model the run sends, once the server has compiled it: a list asked under another name is asked again + const sent = C.servedModelOf(select.dataset.servedAdd, select.dataset.servedShown, compiledNow()); + if (sent !== select.dataset.servedModel) { select.dataset.servedModel = sent; select.length = 1; delete select.dataset.servedFilled; } + if (select.dataset.servedFilled) return; + select.dataset.servedFilled = 'true'; + const model = select.dataset.servedModel; + if (!model) return; + const got = await servedList(model); + if (!select.isConnected || select.dataset.servedModel !== model) return; + const first = select.options[0]; + if (!got.ok || !got.endpoints.length) { first.textContent = got.ok ? 'No provider listed: type a slug' : got.message; return; } + first.textContent = 'Add a provider…'; + for (const e of got.endpoints) { const o = document.createElement('option'); o.value = e.slug; o.textContent = e.name + ' · ' + e.slug + (e.quantization ? ' · ' + e.quantization : ''); select.append(o); } + }); + } function customRows(agent = '') { // '' lists the shared entries (every agent, and the other owners); an id lists that agent's own --ak. @@ -731,7 +770,7 @@ const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec); html='

Review the experiment.

'+rows.map(([i,t,v])=>`
${t}${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'
':'')+runPanel(); } - $('#nb-form').innerHTML=html+'

';applyPick(); + $('#nb-form').innerHTML=html+'

';applyPick();fillServed(); if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();} $('#nb-back').hidden=step===S.benchmark; $('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' →'; @@ -949,6 +988,7 @@ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} + if(d.servedBy!==undefined){C.setServed(rec,d.servedBy,el.value);refreshPlan();return;} if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; const value=el.type==='number'?(el.value===''?'':Number(el.value)):el.value; @@ -958,6 +998,7 @@ else if(!d.providerModel&&['model','userModel','evalModel'].includes(d.field)&&/^(openrouter|openai)\//.test(String(value))){rec.provider=C.providerOf(rec,value);rec[d.field]=C.bareModel(value); renderForm();refreshPlan();const f=root.querySelector(`[data-field="${d.field}"]`);if(f){f.focus();f.setSelectionRange(f.value.length,f.value.length);}return;} // the provider the prefix named, and whether it is set else if(d.field==='provider'){C.setChoice(rec,value);renderForm();refreshPlan();return;} // whether it is set in Settings, and the names a runner's models are sent as, follow the provider + else if(['model','simulator_model','judge_model'].includes(d.field))C.setModelOf(rec,d.field,value); // the field's pin goes with its model else rec[d.field]=value; if(el.list?.id==='nb-paths')completeDirectory(el.value); }else if(d.minutes){C.setPhaseMinutes(rec,d.minutes,el.value===''?'':Number(el.value));const row=el.closest('tr');if(row){const m=row.querySelector('.nb-mono');const t=C.effectiveTimeouts(rec);const p=t&&t.phases.find(x=>x.key===d.minutes);if(m&&p)m.textContent='×'+p.multiplier;}refreshPlan();return;} @@ -979,6 +1020,11 @@ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} + // a provider added to a Served by, after the ones already there; the field shows the list, in its order + else if(d.servedAdd!==undefined){const rec=r(),role=d.servedAdd,now=(rec.served_by||{})[role]||[];if(el.value&&!now.includes(el.value))C.setServed(rec,role,[...now,el.value]);el.value='';const f=root.querySelector(`[data-served-by="${role}"]`);if(f)f.value=((r().served_by||{})[role]||[]).join(', ');refreshPlan();} + // a model chosen: its Served by lists that model's providers; the keyboard stays on the field the new form holds + else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan(); + requestAnimationFrame(()=>{const f=root.querySelector(`[data-field="${d.field}"]`);if(f&&document.activeElement!==f)f.focus();});} else if(d.stdVersion!==undefined){C.setStandardBenchmark(r(),el.value);render();} // an option at its default is left out: the recipe names only what was changed. A select's value can show or hide // the options that apply under it (`when`): the step is drawn again and the focus goes back to the select, or to the diff --git a/app/tests/test_dependency_surface.py b/app/tests/test_dependency_surface.py index 12b6b2e..d6cbd84 100644 --- a/app/tests/test_dependency_surface.py +++ b/app/tests/test_dependency_surface.py @@ -9,6 +9,7 @@ viewer independent of the runtime. """ +import fnmatch import importlib.util import json import os @@ -80,6 +81,26 @@ def test_every_backend_package_is_shipped_once(self): self.assertEqual({p for p in packages if p == "nullius" or p.startswith("nullius.")}, source) self.assertEqual({p.split(".")[0] for p in packages}, {"nullius", "_nullius_app"}) + def test_every_file_a_bundled_definition_runs_is_shipped(self): + """Every file a host's snapshot takes of a bundled entry is package data too.""" + from nullius.standard import payload + shipped = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["setuptools"]["package-data"] + + def covered(package, relative): + parts = relative.split("/") + return any(len(g.split("/")) == len(parts) and all(map(fnmatch.fnmatchcase, parts, g.split("/"))) + for g in shipped[package]) + + base = APP / "nullius" + entries = [("nullius.catalogue", f"{kind}/{e.name}", payload._entry(e)) + for kind in ("agents", "benchmarks", "capabilities") + for e in sorted((base / "catalogue" / kind).iterdir()) if e.is_dir() and e.name != "__pycache__"] + entries += [("nullius.standard", f"kits/{e.name}", payload._entry(e, kit=True)) + for e in sorted((base / "standard" / "kits").iterdir()) if (e / "standard.json").is_file()] + missing = [f"{where}/{name}" for package, where, files in entries for name in files + if not covered(package, f"{where}/{name}")] + self.assertEqual(missing, []) + def test_every_requirement_is_on_the_allowlist(self): for req in requirements(): with self.subTest(req): diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py new file mode 100644 index 0000000..49fcc16 --- /dev/null +++ b/app/tests/test_standard_context_window.py @@ -0,0 +1,318 @@ +"""An agent's last call over the model's window ends the unit as ContextWindowExceededError, whatever followed.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from nullius.runs import classify, parse +from nullius.standard import definitions, install, job, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +WORDS = "the gated task's own sentence" +OPENROUTER = f"HTTP 400: This endpoint's maximum context length is 1000 tokens. However, you requested 1200: {WORDS}" +ANTHROPIC = f"HTTP 400: prompt is too long: 1200 tokens > 1000 maximum: {WORDS}" +EMPTY = "HTTP 400: messages: at least one message is required" +AGENT_OK = {"status": 200} +OVERFLOW = {"status": 400, "error": OPENROUTER} +SIDE_EMPTY = {"status": 400, "error": EMPTY, "side": True} +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's runner ended on its simulated user's error") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' +STOPPED = ANSWERING.replace('raise RuntimeError("the upstream\'s runner ended on its simulated user\'s error")', + 'return {"answer": "DONE"}').replace('{"reward": 0.0, "detail": {}}', + '{"reward": 0.5, "detail": {}, "stopped": "context window"}') +UNREAD = '''from nullius.standard.jsonl import serve +class VerifierError(RuntimeError): + pass +def score(unit, dir, **_): + raise VerifierError("the evaluation did not read the VM") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": lambda unit, dir, setup, model, base_url, key, **_: {"answer": "DONE"}, "score": score}) +''' +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Failure(unittest.TestCase): + def test_the_agents_overflow_outranks_the_side_call_after_it(self): + calls = [AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, agent=True), (relay.CONTEXT_ERROR, OPENROUTER)) + exc, said = relay.failure(calls, words=False, agent=True) + self.assertEqual(exc, "ContextWindowExceededError") + self.assertIn("answered 400", said) + self.assertIn("context window", said) + self.assertNotIn(WORDS, said) + self.assertNotIn("maximum context length", said) + + def test_anthropics_words_and_a_later_rate_limit(self): + calls = [AGENT_OK, {"status": 400, "error": ANTHROPIC}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_the_agents_overflow_as_the_last_call(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW], words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_an_overflow_the_agent_recovered_from_is_not_the_units(self): + self.assertIsNone(relay.failure([OVERFLOW, AGENT_OK], agent=True)) + + def test_a_side_call_that_went_through_after_it_does_not_recover_the_agent(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW, {"status": 200, "side": True}], agent=True)[0], relay.CONTEXT_ERROR) + limited = [AGENT_OK, {"status": 429, "error": "slow down"}, {"status": 502, "error": "x"}, + {"status": 429, "error": "slow down"}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(limited, words=False, agent=True)[0], "RateLimitError") + self.assertIsNone(relay.failure(limited[1:3] + [AGENT_OK, {"status": 200, "side": True}], agent=True)) + self.assertIsNone(relay.failure(limited, words=False)) + calls = [AGENT_OK, OVERFLOW, {"status": 200, "side": True}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + self.assertEqual(relay.failure(calls, words=False)[0], "RateLimitError") + calls = [{"status": 429, "error": "x"}, {"status": 502, "error": "x", "side": True}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "RateLimitError") + + def test_the_calls_are_read_in_the_order_they_began(self): + abandoned = {"status": 502, "error": "x", "t0": 1} + self.assertIsNone(relay.failure([dict(AGENT_OK, t0=2), abandoned], agent=True)) + self.assertEqual(relay.failure([{"status": 429, "error": "x", "t0": 2}, dict(AGENT_OK, t0=1)])[0], "RateLimitError") + self.assertEqual(relay.failure([dict(AGENT_OK, t0=1), {"status": 429, "error": "x"}], agent=True)[0], "RateLimitError") + + def test_another_400_of_the_agent_stays_the_infrastructures(self): + calls = [AGENT_OK, {"status": 400, "error": "HTTP 400: invalid image"}, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_side_calls_overflow_is_not_the_agents(self): + calls = [AGENT_OK, dict(OVERFLOW, side=True)] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_judges_or_simulators_route_is_never_the_agents_context(self): + calls = [AGENT_OK, OVERFLOW] + self.assertEqual(relay.failure(calls, words=False)[0], "ApiError") + self.assertEqual(relay.failure(calls)[0], "ApiError") + + def test_only_a_400_says_so(self): + self.assertTrue(relay.overflowed(OVERFLOW)) + self.assertTrue(relay.overflowed({"status": 400, "error": ANTHROPIC})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: input length and `max_tokens` exceed context " + "limit: 196000 + 8192 > 200000"})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: The input token count (1200) exceeds the " + "maximum number of tokens allowed (1000)."})) + self.assertFalse(relay.overflowed(dict(OVERFLOW, status=500))) + self.assertFalse(relay.overflowed({"status": 400, "error": EMPTY})) + self.assertFalse(relay.overflowed({"status": 400})) + + def test_it_is_the_agents_result_not_the_infrastructure(self): + self.assertEqual(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.AGENT_ERROR) + self.assertNotIn(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.INFRA_OUTCOMES) + fed = job.feed_outcome("score", None, (relay.CONTEXT_ERROR, "said")) + self.assertEqual((fed["outcome"], fed["exc"]), ("agent_error", relay.CONTEXT_ERROR)) + + +class Calls: + """The agent's calls ending on an overflow, then a refused side call.""" + def __init__(self, own, dir=os.devnull): + self.own, self.dir = own, dir + + def calls(self, tag): + return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Unit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "paper") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, setups=SETUPS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def ran(self, own, gated=True, record=os.devnull): + b = dict(self.b, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own, record)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f).get("exception_info"), parse.parse_job(job_dir) + + def test_the_unit_is_the_agents_context_said_by_status_alone(self): + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn("answered 400", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + (trial,) = ran.trials + self.assertEqual(trial.outcome, classify.AGENT_ERROR) + self.assertFalse(trial.is_infra) + self.assertEqual(ran.n_infra, 0) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY], gated=False) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn(WORDS, said["exception_message"]) + + def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW]) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + + def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_a_429(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace('"stopped": "context window"', '"stopped": "batch cap"')) + record = os.path.join(self.tmp, "record") + os.makedirs(record) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("batch cap\n") + limited = {"status": 429, "error": "slow down"} + for calls in ([AGENT_OK, limited], [AGENT_OK, limited, {"status": 502, "error": "down", "side": True}]): + said, ran = self.ran(calls, record=record) + self.assertIsNone(said, calls) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + with open(os.path.join(ran.trials[0].dir, "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + said, _ = self.ran([AGENT_OK, limited]) + self.assertEqual(said["exception_type"], "RateLimitError") + for own in (AGENT_OK, {"status": 401, "error": "no"}, {"status": 502, "error": "down"}): + said, _ = self.ran([own, dict(limited, side=True)], record=record) + self.assertEqual(said["exception_type"], "RateLimitError", own) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("context window\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + with open(os.path.join(record, relay.STOP), "a") as f: + f.write("batch cap\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace(', "stopped": "context window"', "")) + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD) + said, ran = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "VerifierError") + self.assertIn("the evaluation did not read the VM", said["exception_message"]) + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD.replace("raise VerifierError", "raise RuntimeError")) + said, _ = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + + def test_a_unit_run_again_under_its_jobs_name_starts_with_no_earlier_record_and_keeps_a_new_stop(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + record = os.path.join(jobs, "again", "a__u0000", ".work", "relay") + os.makedirs(record) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "verifier")) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "agent")) + for name, text in (("u0000.jsonl", json.dumps({"status": 429, "error": "slow down"}) + "\n"), (relay.STOP, "batch cap\n"), + ("../../verifier/score.json", "{}"), ("../../result.json", "{}"), ("../../agent/old.txt", "")): + with open(os.path.join(record, name), "w") as f: + f.write(text) + seen = [] + + def phase(r, name, unit, phase, timeout=None): + if phase == "env_setup": + os.makedirs(record, exist_ok=True) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("key cap\n") + + def unit_relay(r, tag, trial, *a, **k): + with open(os.path.join(trial, ".work", "relay", relay.STOP)) as f: + seen.append((sorted(os.listdir(os.path.join(trial, ".work", "relay"))), f.read())) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "phase", phase), patch.object(run.Run, "unit_relay", unit_relay): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + self.assertEqual(seen, [([relay.STOP], "key cap\n")]) + (earlier,) = [d for d in os.listdir(os.path.dirname(record)) if d.startswith("earlier.")] + earlier = os.path.join(os.path.dirname(record), earlier) + with open(os.path.join(earlier, "relay", relay.STOP)) as f: + self.assertEqual(f.read(), "batch cap\n") + for part in (("verifier", "score.json"), ("result.json",), ("agent", "old.txt")): + self.assertTrue(os.path.isfile(os.path.join(earlier, *part))) + self.assertFalse(os.path.exists(os.path.join(jobs, "again", "a__u0000", "verifier", "score.json"))) + + def test_a_run_under_a_job_name_another_run_holds_is_refused_before_it_writes(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + job_dir = os.path.join(jobs, "again") + lck = os.path.join(job_dir, ".work", "job.lck") + tried = subprocess.run([sys.executable, "-c", "import fcntl, sys\nfcntl.flock(open(sys.argv[1], 'a'), fcntl.LOCK_EX | fcntl.LOCK_NB)", lck], + capture_output=True) + self.assertNotEqual(tried.returncode, 0) + run.HELD.pop(lck).close() + before = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time\nf = open(sys.argv[1], 'a')\n" + "fcntl.flock(f, fcntl.LOCK_EX)\nprint(flush=True)\ntime.sleep(30)", + lck], stdout=subprocess.PIPE) + self.addCleanup(holder.stdout.close) + self.addCleanup(holder.kill) + holder.stdout.readline() + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])), \ + self.assertRaisesRegex(run.Refused, "another run under the job name again is running"): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + after = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + self.assertEqual(after, before) + + def test_a_second_run_in_the_same_process_under_a_live_job_name_is_refused(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + go = lambda: run.run(self.b, o, model="paper-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + name="live", say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + tried = [] + + def relay(r, *a, **k): + if not tried: + tried.append(None) + for again in (go, r.start): + try: + tried.append(again()) + except run.Refused as e: + tried.append(e) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "unit_relay", relay): + go() + self.assertEqual(len(tried), 3) + for refused in tried[1:]: + self.assertIsInstance(refused, run.Refused) + self.assertIn("another run under the job name live is running", str(refused)) + go() + + def test_a_side_call_failing_after_the_agents_success_stays_the_infrastructures(self): + said, ran = self.ran([AGENT_OK, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ApiError") + self.assertEqual(ran.trials[0].outcome, classify.API_ERROR) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index e1c6b84..7dd4f93 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -166,6 +166,107 @@ def test_a_standard_job_named_oracle_without_the_record_is_no_control(self): self.assertFalse(parse.parse_job(j.dir).is_control) # a runner's arm called oracle stays an arm +SLOW_SCORE = '''import os, time +from nullius.standard.jsonl import serve +CHECKED = os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "checked")) +def setup(unit, dir): + return dict({"prompt": "hi"}, **({"verify": {"run": "true"}} if CHECKED else {})) +def score(unit, dir, transcript, answer, verified=None): + time.sleep(2) + return {"reward": 0.0, "passed": False, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "score": score}) +''' + + +class Box: + image = "" + + def give_back(self, paths): + pass + + def stop(self): + pass + + +class ScoreLimit(unittest.TestCase): + """A null control whose driver does its work in ``score``, waited for ``limits.score_seconds``.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_SCORE) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def problems(self, limits): + return definitions.problems(dict(self.b, limits=limits), "benchmark") + + def trial(self, limits): + b = dict(self.b, limits=limits) + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", + jobs_root=os.path.join(self.tmp, "jobs"), env={}, say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f) + + def verify_phase(self): + jobs = os.path.join(self.tmp, "jobs") + with open(os.path.join(jobs, os.listdir(jobs)[0], "events.jsonl")) as f: + return next(e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "verify") + + def test_the_limit_is_a_whole_number_of_seconds_from_1(self): + self.assertEqual(self.problems({"score_seconds": 1800}), []) + for v in (0, -1, 1.5, "600", True, 100001): + self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + + def test_run_again_is_true_or_false(self): + for v in (False, True, None): + self.assertEqual(self.problems({"run_again": v}), [], v) + for v in (0, "no"): + self.assertIn("limits.run_again: false when a rate-limited unit is not run once more", self.problems({"run_again": v}), v) + + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): + r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) + self.assertEqual(r.score_seconds, 600) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}) + self.assertEqual(r.score_seconds, 1800) + + def test_a_score_past_the_limit_is_the_verifiers_error_and_one_within_it_scores(self): + late = self.trial({"score_seconds": 1}) + self.assertEqual(late["exception_info"]["exception_type"], "VerifierError") + self.assertIn("score: no reply in 1 s", late["exception_info"]["exception_message"]) + self.assertIsNone(late.get("verifier_result")) + on_time = self.trial({"score_seconds": 10}) + self.assertIsNone(on_time.get("exception_info")) + self.assertEqual(on_time["verifier_result"]["rewards"]["reward"], 0.0) + + def test_the_verify_phase_carries_the_score_limit_and_an_own_images_check(self): + self.trial({"score_seconds": 10}) + self.assertEqual(self.verify_phase()["timeout_sec"], 10) + shutil.rmtree(os.path.join(self.tmp, "jobs")) + open(os.path.join(self.b["_here"], "checked"), "w").close() + checked = [] + with patch.object(run.Run, "unit_environment", lambda r, setup: ("task", {"image": "own"})), \ + patch.object(run.Run, "not_run", lambda r, e, d: None), \ + patch.object(run.Run, "unit_relay", lambda r, *a, **k: None), \ + patch.object(run.Run, "own_box", lambda r, *a: Box()), \ + patch.object(run.Run, "box_changes", lambda r, box: ""), \ + patch.object(run.Run, "release_image", lambda r, image: None), \ + patch.object(run.Run, "verify", lambda r, step, box, trial: checked.append(step) or {"exit": 0}): + got = self.trial({"score_seconds": 10}) + self.assertIsNone(got.get("exception_info")) + self.assertEqual(checked, [{"run": "true"}]) + self.assertEqual(self.verify_phase()["timeout_sec"], 10 + run.TURN_SECONDS) + + class Prove(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() @@ -281,6 +382,254 @@ def test_units_that_may_not_be_redistributed_are_counted_never_named(self): self.assertIn("held-unit-2", b["proven"]["leaky"]["failed"]["leak"]["problems"][0]) +class ProveOptions(unittest.TestCase): + """The benchmark's options reach the checks' driver and each control's run; a merged control job is held as one.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "kits", "agent-task-v1") + shutil.copytree(os.path.join(definitions.KITS, "agent-task-v1"), here) + with open(os.path.join(here, "driver.py")) as f: + src = f.read() + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(src.replace('return {"units": list(WORDS)}', + 'import os\n return {"units": list(WORDS) if os.environ.get("NULLIUS_OPTION_SITES") == "on" else []}')) + with open(os.path.join(here, "standard.json")) as f: + d = json.load(f) + d["options"] = {"sites": {"summary": "The sites.", "values": ["off", "on"], "default": "off"}, + "lane": {"summary": "A lane.", "values": ["a", "b"], "default": "a"}, + "tool": {"summary": "A tool.", "values": ["none", "x"], "default": "none", + "needs": {"x": {"host": ["tool"]}}}} + d["host"] = {"tool": {"summary": "A program this machine lacks.", "check": ["nullius-no-such-program-k5"]}} + with open(os.path.join(here, "standard.json"), "w") as f: + json.dump(d, f) + self.b, bad = definitions._read(os.path.join(here, "standard.json"), "benchmark", "agent-task-v1") + assert not bad, bad + self.jobs = os.path.join(self.tmp, "jobs") + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def prove(self, **kw): + kw.setdefault("stamp", "2026-01-02__03-04-05") + return prove.prove(self.b, jobs_root=self.jobs, env={}, say=lambda s: None, **kw) + + def config(self, job_dir): + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"] + + def nop_shard(self, i, n, lane): + return run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=self.jobs, env={}, + say=lambda s: None, shard=(i, n), launch="L", bench_options={"sites": "on", "lane": lane}) + + def test_the_options_reach_the_units_driver_and_each_controls_run(self): + got = self.prove(bench_options={"sites": "on"}, write=False)["kit"] + self.assertTrue(all(r["ok"] for r in got.values()), got) + self.assertEqual((got["units"]["units"], got["nop"]["passed"], got["oracle"]["passed"]), (3, "0/3", "3/3")) + self.assertEqual(self.config(os.path.join(self.jobs, got["nop"]["job"]))["benchmark_options"], + {"sites": "on", "lane": "a", "tool": "none"}) + self.assertEqual(got["nop"]["options"], {"sites": "on", "lane": "a", "tool": "none"}) + dflt = self.prove(write=False, stamp="2026-01-02__03-04-06")["kit"] + self.assertEqual(dflt["units"]["problems"], ["units: no ids"]) + self.assertEqual(dflt["nop"]["problems"], ["the units check failed: the control was not run"]) + + def test_a_name_or_a_value_the_benchmark_does_not_allow_is_refused_before_anything_runs(self): + was = json.dumps(self.b.get("proven")) + for given, why in (({"sites": "maybe"}, "'maybe' is not one of off, on"), ({"nope": "x"}, "has no option 'nope'")): + with self.assertRaises(run.Refused) as e: + self.prove(bench_options=given) + self.assertIn(why, str(e.exception)) + self.assertFalse(os.path.exists(self.jobs)) + self.assertEqual(json.dumps(self.b.get("proven")), was) + + def test_an_option_whose_value_needs_a_program_this_machine_lacks_is_refused_before_any_install(self): + with patch.object(install, "upstream", side_effect=AssertionError("installed")) as up: + with self.assertRaises(run.Refused) as e: + self.prove(bench_options={"sites": "on", "tool": "x"}) + self.assertIn("nullius-no-such-program-k5 not on PATH", str(e.exception)) + up.assert_not_called() + self.assertFalse(os.path.exists(self.jobs)) + + def test_a_job_is_not_recorded_when_the_units_check_fails(self): + from nullius.standard import merge + whole = merge.merge([self.nop_shard(i, 4, "a") for i in (1, 3)], os.path.join(self.jobs, "nop-merged"))["out"] + self.b["datasets"][0]["units"] = 4 + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertFalse(got["units"]["ok"]) + self.assertFalse(got["nop"]["ok"]) + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", got["nop"]["problems"]) + kept = self.b["proven"]["kit"] + self.assertNotIn("nop", kept) + self.assertNotIn("units", kept) + self.assertEqual(kept["failed"]["nop"]["job"], "nop-merged") + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", + kept["failed"]["nop"]["problems"]) + + def test_a_control_is_not_run_nor_kept_when_the_units_check_fails(self): + self.assertIn("nop", self.b["proven"]["kit"]) + self.b["datasets"][0]["units"] = 4 + got = self.prove(bench_options={"sites": "on"})["kit"] + self.assertFalse(got["units"]["ok"]) + for c in ("nop", "oracle"): + self.assertEqual(got[c]["problems"], ["the units check failed: the control was not run"]) + self.assertNotIn(c, self.b["proven"]["kit"]) + self.assertIn(c, self.b["proven"]["kit"]["failed"]) + self.assertFalse(os.path.exists(self.jobs) and any("prove-" in x for x in os.listdir(self.jobs))) + + def test_a_trial_of_another_job_cannot_be_merged_into_a_proof(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, "a") for i in (1, 3)] + trial = next(t for t in sorted(os.listdir(shards[0])) if os.path.isfile(os.path.join(shards[0], t, "result.json"))) + path = os.path.join(shards[0], trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + with self.assertRaises(merge.Refused) as e: + merge.merge(shards, os.path.join(self.jobs, "never")) + self.assertIn("names another job than its job", str(e.exception)) + forced = merge.merge(shards, os.path.join(self.jobs, "forced"), force=True)["out"] + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": forced}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_a_merge_of_merges_keeps_what_any_of_them_was_merged_despite(self): + from nullius.standard import merge + good_shard, bad_shard = (self.nop_shard(i, 4, "a") for i in (1, 3)) + trial = next(t for t in sorted(os.listdir(bad_shard)) if os.path.isfile(os.path.join(bad_shard, t, "result.json"))) + path = os.path.join(bad_shard, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + good = merge.merge([good_shard], os.path.join(self.jobs, "good"))["out"] + bad = merge.merge([bad_shard], os.path.join(self.jobs, "bad"), force=True)["out"] + with self.assertRaises(merge.Refused): + merge.merge([good, bad], os.path.join(self.jobs, "never")) + both = merge.merge([good, bad], os.path.join(self.jobs, "both"), force=True)["out"] + std = self.config(both) + self.assertTrue(any(w.startswith("bad: ") and "not of the job they are in" in w for w in std["merged_despite"])) + self.assertEqual([m.get("merged_from", [{}])[0].get("job") for m in std["merged_from"]], [os.path.basename(good_shard), os.path.basename(bad_shard)]) + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": both}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_the_cli_hands_its_options_and_jobs_to_the_proof(self): + from nullius import progress + from nullius.standard import cli + seen = {} + with patch.object(cli, "_bench", return_value=self.b), patch.object(cli, "licences", return_value=(None, None)), \ + patch.object(cli, "_exit_on_term"), patch.object(progress, "ECHO", None), \ + patch.object(prove, "prove", side_effect=lambda b, **kw: seen.update(kw) or {}): + self.assertEqual(cli.main(["prove", "agent-task-v1", "--from-run", "--dataset", "kit", "--bench-option", "sites=on", + "--bench-option", "lane=b", "--job", "nop=/x/merged"]), 0) + self.assertEqual((seen["bench_options"], seen["jobs"]), ({"sites": "on", "lane": "b"}, {"nop": "/x/merged"})) + with self.assertRaises(SystemExit): + cli.main(["prove", "agent-task-v1", "--from-run", "--job", "reference=/x"]) + + def test_a_merged_launch_is_held_to_the_bound_as_one_job_and_must_cover_every_unit(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, lane) for i, lane in ((1, "a"), (3, "b"))] + whole = merge.merge(shards, os.path.join(self.jobs, "nop-merged"))["out"] + before = sorted(os.listdir(self.jobs)) + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertTrue(got["nop"]["ok"], got["nop"]) + self.assertEqual((got["nop"]["job"], got["nop"]["units"], got["nop"]["passed"]), ("nop-merged", 3, "0/3")) + self.assertEqual(self.b["proven"]["kit"]["nop"]["job"], "nop-merged") + self.assertEqual(self.b["proven"]["kit"]["nop"]["options"], + [{"lane": "a", "sites": "on", "tool": "none"}, {"lane": "b", "sites": "on", "tool": "none"}]) + self.assertEqual(sorted(set(os.listdir(self.jobs)) - set(before)), ["2026-01-02__03-04-05__agent-task-v1__kit__prove-oracle"]) + self.assertEqual([m["benchmark_options"]["lane"] for m in self.config(whole)["merged_from"]], ["a", "b"]) + half = merge.merge(shards[:1], os.path.join(self.jobs, "nop-half"))["out"] + part = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": half}, write=False, stamp="2026-01-02__03-04-06")["kit"]["nop"] + self.assertFalse(part["ok"]) + self.assertTrue(any("units have no trial" in p for p in part["problems"]), part["problems"]) + wrong = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"oracle": whole}, write=False, stamp="2026-01-02__03-04-07")["kit"]["oracle"] + self.assertIn("nop-merged: its control is 'nop', not 'oracle'", wrong["problems"]) + trial = next(t for t in sorted(os.listdir(whole)) if os.path.isfile(os.path.join(whole, t, "result.json"))) + with open(os.path.join(whole, trial, "result.json")) as f: + res = json.load(f) + res.pop("verifier_result", None) + res.pop("exception_info", None) + with open(os.path.join(whole, trial, "result.json"), "w") as f: + json.dump(res, f) + blank = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole}, write=False, + stamp="2026-01-02__03-04-08")["kit"]["nop"] + self.assertFalse(blank["ok"]) + self.assertIn("1 unit(s) ended with no reward from 0 to 1 and no error: read the job's trials", blank["problems"]) + for kw in ({"limit": 1}, {"shard": (1, 2)}): + with self.assertRaises(run.Refused): + self.prove(dataset="kit", jobs={"nop": whole}, **kw) + with self.assertRaises(run.Refused): + self.prove(jobs={"nop": os.path.join(self.tmp, "absent")}) + + +NUMERIC = {"key": "numeric", "_ref": "numeric", "label": "Numeric", "family": "agent", "needs": ["agent.task/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "score", "says": "a number from 0 to 1"}, + "controls": {"nop": {"source": "nullius", "from": "Nothing runs."}}, "datasets": [{"id": "all", "units": 3}]} + + +class HeldJob(unittest.TestCase): + """A control's job held to its bound, read from each trial's own record.""" + + def setUp(self): + from nullius.standard import job + self.tmp = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, self.tmp, True) + self.dir = os.path.join(self.tmp, "nop-job") + os.makedirs(self.dir) + nop = definitions.control(NUMERIC, "nop") + self.units = ["u1", "u2", "u3"] + job.write_config(self.dir, job="nop-job", bench=NUMERIC, other=nop, model="", provider="openai", dataset="all", + units=self.units, concurrency=1, granted=False) + results = [job.write_trial(os.path.join(self.dir, f"{u}__u{i:04d}"), job="nop-job", bench=NUMERIC, other=nop, model="", + unit=u, t0=1_760_000_000.0, t1=1_760_000_001.0, produce=(None, None), score={"reward": 0.0}, + exception=None, calls=[], trajectory=None) for i, u in enumerate(self.units)] + job.write_result(self.dir, "nop-job", 1_760_000_000.0, 1_760_000_002.0, results, 3) + + def held(self): + return prove.control_check(NUMERIC, NUMERIC["datasets"][0], "nop", limit=None, jobs_root=self.tmp, stamp="s", + provider="openai", model="", judge_model=None, env={}, concurrency=1, + say=lambda s: None, job_dir=self.dir, ids=self.units) + + def edit(self, unit, change): + trial = next(t for t in os.listdir(self.dir) if t.startswith(unit + "__")) + path = os.path.join(self.dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + change(res) + with open(path, "w") as f: + json.dump(res, f) + + def test_a_clean_job_passes(self): + r = self.held() + self.assertTrue(r["ok"], r["problems"]) + self.assertEqual((r["passed"], r["value"], r["units"]), ("0/3", 0.0, 3)) + + def test_a_reward_without_a_score_a_score_below_0_and_nan_are_no_score(self): + for value in ({"reward": 1.0}, {"score": -100}, {"score": float("nan")}, {"score": True}, {"score": "0"}): + self.edit("u2", lambda res: res["verifier_result"].update(rewards=value)) + r = self.held() + self.assertFalse(r["ok"], value) + self.assertIn("1 unit(s) ended with no score from 0 to 1 and no error: read the job's trials", r["problems"], value) + self.assertEqual(r["value"], 0.0) + + def test_a_trial_another_run_wrote_is_refused(self): + self.edit("u1", lambda res: res["standard"].update(benchmark="other", control={"name": "oracle"}, plugged="oracle")) + self.edit("u3", lambda res: res["standard"].update(job="another-job")) + r = self.held() + self.assertFalse(r["ok"]) + said = " | ".join(r["problems"]) + for k in ("benchmark", "control", "who ran it", "job"): + self.assertIn(f"whose own record names another {k} than the job", said) + + LEAKY = '''from nullius.standard.jsonl import serve import os ANSWER = "the reference answer is forty-two" diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py new file mode 100644 index 0000000..17506a1 --- /dev/null +++ b/app/tests/test_standard_gated.py @@ -0,0 +1,134 @@ +"""A gated benchmark's failing endpoint is said by its status alone, down to the trial's result.json.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius.standard import definitions, install, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +CONTROLS = {"nop": {"source": "nullius", "from": "Nothing runs; the answer is empty."}} +WORDS = "the gated task's own sentence" +FAILING = '''from nullius.standard.jsonl import serve +def score(unit, dir, transcript, answer, **_): + raise RuntimeError("the LLM grader's endpoint answered 402, and the task was scored without it") +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, "score": score}) +''' + + +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's grader said: status 402: " + %r) +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' % WORDS +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Calls: + """The unit's relay as Run.unit reads it: each route's calls, the judge's ending on a refusal with words.""" + def __init__(self, side): + self.side, self.dir = side, os.devnull + + def calls(self, tag): + if tag.endswith("-" + self.side): + return [{"status": 200}, {"status": 402, "error": f"status 402: {WORDS}"}] + return [] + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Gated(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "failing") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(FAILING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def saved(self, side, **bench): + b = dict(self.b, **bench) + jobs = os.path.join(self.tmp, "jobs-" + side + str(bench)) + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", jobs_root=jobs, env={}, + say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_refused_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.saved(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said = self.saved("judge") + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn(WORDS, said["exception_message"]) + + def answered(self, side, gated): + """drive_upstream's answer failing on its judge's or simulated user's refusal.""" + here = os.path.join(self.tmp, "paper") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + b = dict(self.b, _here=here, setups=SETUPS, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"answer-{side}-{gated}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_answer_failing_on_its_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.answered(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_an_answer_failing_on_a_benchmark_not_gated_keeps_its_words(self): + said = self.answered("judge", gated=False) + self.assertEqual(said["exception_type"], "NonZeroAgentExitCodeError") + self.assertIn(WORDS, said["exception_message"]) + + def test_the_failure_alone(self): + calls = [{"status": 429, "error": WORDS}] + self.assertEqual(relay.failure(calls), ("RateLimitError", WORDS)) + exc, said = relay.failure(calls, words=False) + self.assertEqual(exc, "RateLimitError") + self.assertNotIn(WORDS, said) + self.assertEqual(relay.failure([{"status": 404, "error": "no such model"}], words=False)[0], "ModelNotFoundError") + + def test_gated_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) + self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", + definitions.problems(dict(self.b, gated="yes"), "benchmark")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_parts.py b/app/tests/test_standard_parts.py index ea875b8..318af12 100644 --- a/app/tests/test_standard_parts.py +++ b/app/tests/test_standard_parts.py @@ -35,6 +35,15 @@ def trial(self, name, f1s): exception=None, calls=[], trajectory=None) return d + def test_a_stopped_score_says_so_in_the_result(self): + for name, score in (("stopped", {"reward": 0.5, "stopped": "batch cap"}), ("whole", {"reward": 0.5})): + job.write_trial(os.path.join(self.job, name), job="job", bench=BENCH, other={"key": "original", "_kind": "upstream"}, + model="m", unit=name, t0=0, t1=1, produce=(0, 1), score=score, exception=None, calls=[], trajectory=None) + with open(os.path.join(self.job, "stopped", "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + with open(os.path.join(self.job, "whole", "result.json")) as f: + self.assertNotIn("stopped", json.load(f)["verifier_result"]) + def test_the_mean_is_over_every_question(self): self.trial("conv-a", [1.0] * 4) # 4 questions, all right self.trial("conv-b", [0.0] * 12) # 12 questions, all wrong diff --git a/app/tests/test_standard_recipe_served_by.py b/app/tests/test_standard_recipe_served_by.py new file mode 100644 index 0000000..a24a99e --- /dev/null +++ b/app/tests/test_standard_recipe_served_by.py @@ -0,0 +1,180 @@ +"""The builder's Served by: a recipe's ``served_by`` (the serving providers +each model field pins, on OpenRouter) is checked as ``--served-by`` is and +compiled into it for each job that calls that model, by the name the run +sends it as; a recipe with none compiles to the command it did. The model's +providers come from OpenRouter's endpoint list, asked as the model list is.""" +from __future__ import annotations + +import io +import json +import os +import unittest +import urllib.error + +from nullius import providers +from nullius.standard import definitions, recipe + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +NOP = {"nop": {"source": "nullius", "from": "nothing said"}} + + +def defs(bench=None): + d = {k: definitions.load(k, ROOT)[0] for k in ("benchmark", "agent", "capability")} + if bench is not None: + d["benchmark"] = [bench if x["key"] == bench["key"] else x for x in d["benchmark"]] + return d + + +def fakebench(**kw): + return dict(definitions.find(definitions.load("benchmark", ROOT)[0], "fakebench"), **kw) + + +def judged(): + b = fakebench(controls=NOP) + return dict(b, score=dict(b["score"], judge={"model": "gpt-4o-mini", "says": "a judge"})) + + +def a_recipe(**kw): + r = {"version": 1, "harness": "standard", "name": "fake-0101-120000", "title": "t", "benchmark": "fakebench", "dataset": "all", + "other": "fake", "provider": "openrouter", "model": "openai/gpt-5-mini", "limit": 2, "concurrency": 2, "unsandboxed": True, + "where": "local"} + r.update(kw) + return r + + +class TheRecipe(unittest.TestCase): + def test_a_pin_compiles_to_the_option_and_shows_in_the_review(self): + out = recipe.compile(a_recipe(served_by={"model": ["deepinfra", "together/fp8"]}), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--served-by openai/gpt-5-mini=deepinfra,together/fp8", out["script"]) + model_row = next(row[2] for row in out["review"] if row[1] == "Model") + self.assertIn("model served by deepinfra, together/fp8", model_row) + + def test_each_job_is_pinned_where_it_calls_the_model(self): + """A control beside the agent calls no model of its own: its job takes no pin for the run's model; a judge + pin goes on every job of a judged benchmark, the control's included, by the name OpenRouter serves it as.""" + d = defs(judged()) + out = recipe.compile(a_recipe(others=["fake", "nop"], served_by={"model": ["deepinfra"], "judge": ["prov-j"]}), d) + self.assertEqual(out["errors"], []) + scripts = [p["script"] for p in out["parts"]] + self.assertIn("--served-by openai/gpt-5-mini=deepinfra", scripts[0]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[0]) + self.assertNotIn("openai/gpt-5-mini=deepinfra", scripts[1]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[1]) + row = next(r for r in out["review"] if r[1] == "Model")[2] + self.assertIn("judge served by prov-j", row) + + def test_a_pin_is_refused_as_the_option_is(self): + for kw, said in (({"provider": "openai", "model": "gpt-5-mini", "served_by": {"model": ["deepinfra"]}}, "needs OpenRouter"), + ({"served_by": {"model": ["bad slug"]}}, "not a serving provider's slug"), + ({"served_by": {"model": ["deepinfra", "deepinfra"]}}, "named twice"), + ({"served_by": {"agent": ["deepinfra"]}}, "Served by names"), + ({"served_by": ["deepinfra"]}, "Served by names"), + ({"served_by": {"judge": ["prov-j"]}}, "no job of this recipe calls the judge's model"), + ({"served_by": {"model": "deepinfra"}}, "Served by")): + with self.subTest(kw=kw): + out = recipe.compile(a_recipe(**kw), defs()) + self.assertTrue(out["errors"], out["errors"]) + self.assertTrue(all(e["step"] == recipe.STEP["model"] for e in out["errors"]), out["errors"]) + self.assertTrue(any(said in e["message"] for e in out["errors"]), out["errors"]) + self.assertEqual(out["script"], "") + + def test_one_model_in_two_fields_takes_one_list(self): + d = defs(judged()) + same = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["a"]}), d) + self.assertEqual(same["errors"], []) + lines = [line for line in same["script"].splitlines() if "--served-by" in line] + self.assertTrue(lines and all(line.count("--served-by") == 1 for line in lines)) # each way of starting it: once + other = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["b"]}), d) + self.assertIn("pins openai/gpt-4o-mini twice", other["errors"][0]["message"]) + + def test_a_pin_is_its_models_on_every_job_that_calls_it(self): + """The model field's model is also the judge's: the control beside the agent calls it as its judge, and its job + takes the pin too.""" + out = recipe.compile(a_recipe(others=["fake", "nop"], model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), defs(judged())) + self.assertEqual(out["errors"], []) + for part in out["parts"]: + self.assertIn("--served-by openai/gpt-4o-mini=prov-a", part["script"]) + + def test_a_list_with_an_empty_or_a_bad_slug_is_refused_in_sight(self): + for slugs in (["deepinfra", ""], ["deepinfra", "bad slug"]): + with self.subTest(slugs=slugs): + out = recipe.compile(a_recipe(served_by={"model": slugs}), defs()) + self.assertTrue(any("not a serving provider's slug" in e["message"] for e in out["errors"]), out["errors"]) + + def test_each_fields_model_is_given_as_the_run_sends_it(self): + """Like the paper, LongMemEval's gpt-4o is sent as openai/gpt-4o-2024-08-06 on OpenRouter: its providers are that + model's, and the pin is put on that name.""" + sets = definitions.find(definitions.load("benchmark")[0], "longmemeval")["datasets"] + r = a_recipe(benchmark="longmemeval", dataset=sets[0]["id"], other="", others=["original"], model_choice="paper", model="gpt-4o", + served_by={"model": ["openai"]}) + out = recipe.compile(r) + sent = out["served_models"]["model"] + self.assertEqual(sent, "openai/gpt-4o-2024-08-06") + self.assertIn(f"--served-by {sent}=openai", out["script"]) + self.assertEqual(recipe.compile(a_recipe(), defs())["served_models"]["model"], "openai/gpt-5-mini") + + def test_a_recipe_with_no_pin_compiles_to_the_command_it_did(self): + plain = recipe.compile(a_recipe(), defs()) + self.assertEqual(plain["errors"], []) + self.assertNotIn("--served-by", plain["script"]) + for none in ({}, {"model": []}, None): + with self.subTest(none=none): + self.assertEqual(recipe.compile(a_recipe(served_by=none), defs())["script"], plain["script"]) + openai = recipe.compile(a_recipe(provider="openai", model="gpt-5-mini", served_by={}), defs()) + self.assertEqual(openai["errors"], []) + + +class TheProviders(unittest.TestCase): + """``providers.list_endpoints``: OpenRouter's endpoint list, a fake opener in the place of the network.""" + + def opener(self, body=None, error=None): + seen = [] + + def opener(req, timeout=None): + seen.append(req) + if error: + raise error + return io.BytesIO(json.dumps(body).encode()) + return opener, seen + + def test_a_models_providers_by_slug_and_name(self): + body = {"data": {"id": "meta-llama/llama-3.3-70b-instruct", "endpoints": [ + {"provider_name": "DeepInfra", "tag": "deepinfra/fp8", "quantization": "fp8"}, + {"provider_name": "Together", "tag": "together"}, + {"provider_name": "Google AI Studio"}, # no tag: its name as a slug reads + {"provider_name": "Bad", "tag": "bad slug"}, "not a row", {"tag": 7}]}} + opener, seen = self.opener(body) + got = providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + self.assertEqual(got, [{"slug": "deepinfra/fp8", "name": "DeepInfra", "quantization": "fp8"}, + {"slug": "google-ai-studio", "name": "Google AI Studio", "quantization": ""}, + {"slug": "together", "name": "Together", "quantization": ""}]) + (req,) = seen + self.assertEqual(req.full_url, "https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints") + self.assertIsNone(req.get_header("Authorization")) # no key set: the list is asked without one + self.assertEqual(req.get_method(), "GET") + + def test_the_key_goes_as_for_the_model_list_and_a_bare_name_gets_its_maker(self): + opener, seen = self.opener({"data": {"endpoints": []}}) + providers.list_endpoints("openrouter", "gpt-4o-mini", env={"OPENROUTER_API_KEY": "or-test"}, opener=opener) + self.assertEqual(seen[0].full_url, "https://openrouter.ai/api/v1/models/openai/gpt-4o-mini/endpoints") + self.assertEqual(seen[0].get_header("Authorization"), "Bearer or-test") + + def test_what_cannot_be_asked_or_read_is_said_in_fixed_words(self): + for kw in ({"error": urllib.error.URLError("refused at https://openrouter.ai?key=x")}, {"body": {"error": "no"}}, + {"body": {"data": {"endpoints": "none"}}}): + with self.subTest(kw=kw): + opener, _ = self.opener(**kw) + with self.assertRaisesRegex(ValueError, r"^Provider list unavailable\. Enter a provider's slug by hand\.$"): + providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + opener, seen = self.opener({}) + with self.assertRaisesRegex(ValueError, "Only OpenRouter"): + providers.list_endpoints("openai", "gpt-4o", env={"OPENAI_API_KEY": "x"}, opener=opener) + for model in ("no-maker", "a/b/../../c", "maker/model?x=1", ""): + with self.subTest(model=model), self.assertRaisesRegex(ValueError, "maker/model"): + providers.list_endpoints("openrouter", model, env={}, opener=opener) + self.assertEqual(seen, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 8319794..d6748e4 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -3,6 +3,7 @@ A driver or capability process answers one JSON line per request.""" from __future__ import annotations +import base64 import http.client import json import os @@ -34,6 +35,22 @@ b'{"type":"message","content":[{"type":"output_text","text":"hi"}]}],' b'"usage":{"input_tokens":20,"output_tokens":5,"input_tokens_details":{"cached_tokens":10}}}}\n\n') +RESP_FAILED = json.dumps({"object": "response", "id": "gen-1", "status": "failed", "output": [], "usage": None, + "error": {"code": "server_error", "message": "the request said x"}}).encode() +RESP_FAILED_SSE = (b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","output":[],' + b'"error":{"code":"server_error","message":"the request said x"}}}\n\n') +RESP_INCOMPLETE = json.dumps({"object": "response", "status": "incomplete", "output": [], + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 30, "output_tokens": 8}}).encode() + +HOLLOW = {"hollow": json.dumps({"id": "gen-1", "provider": "Minimax", "model": "m"}).encode(), + "hollow-error": json.dumps({"id": "gen-1", "error": {"code": 502, "message": "the request said x"}}).encode(), + "hollow-429": json.dumps({"id": "gen-1", "error": {"code": 429, "message": "the request said x"}}).encode(), + "hollow-402": json.dumps({"id": "gen-1", "error": {"code": 402, "message": "the request said x"}}).encode(), + "hollow-404": json.dumps({"id": "gen-1", "error": {"code": 404, "message": "the request said x"}}).encode(), + "count": json.dumps({"input_tokens": 12}).encode(), "no-choice": json.dumps({"choices": []}).encode(), + "not-json": b"the request said x"} + class Upstream(BaseHTTPRequestHandler): seen = [] @@ -51,6 +68,23 @@ def do_POST(self): self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") elif mode == "no-usage": self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") + elif mode in ("400", "413", "422"): + self.reply(int(mode), b'{"error":{"message":"could not process image"}}', "application/json") + elif mode in ("cut", "short"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header(*(("Transfer-Encoding", "chunked") if mode == "cut" else ("Content-Length", str(len(CHAT_SSE) + 9)))) + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(CHAT_SSE), CHAT_SSE) if mode == "cut" else CHAT_SSE) + elif mode in HOLLOW: + self.reply(200, HOLLOW[mode], "application/json", *[("X-Generation-Id", "gen-1")] * (mode == "not-json"), + ("Retry-After", "30")) + elif self.path.endswith("/responses") and mode == "failed": + self.reply(200, RESP_FAILED, "application/json") + elif self.path.endswith("/responses") and mode == "failed-sse": + self.reply(200, RESP_FAILED_SSE, "text/event-stream") + elif self.path.endswith("/responses") and mode == "incomplete": + self.reply(200, RESP_INCOMPLETE, "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -58,10 +92,11 @@ def do_POST(self): else: self.reply(200, json.dumps(CHAT).encode(), "application/json") - def reply(self, status, body, kind): + def reply(self, status, body, kind, *headers): try: self.send_response(status) - self.send_header("Content-Type", kind) + for header in (("Content-Type", kind),) + headers: + self.send_header(*header) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -90,6 +125,7 @@ def post(self, tag, path, body): with urllib.request.urlopen(req) as r: return r.status, r.read() except urllib.error.HTTPError as e: + self.retry_after = e.headers.get("Retry-After") return e.code, e.read() def test_plain_reply_passes_through_and_the_real_key_goes_out(self): @@ -120,6 +156,68 @@ def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): self.assertEqual(rec["reply"]["text"], "hello") self.assertEqual(rec["reply"]["usage"]["input"], 7) + def test_a_stream_cut_off_after_its_headers_is_a_failed_call(self): + for mode in ("cut", "short"): + self.post("u" + mode, "/chat/completions", {"model": "m", "stream": True, "mode": mode, "messages": []}) + rec = self.relay.calls("u" + mode)[-1] + self.assertEqual(rec["status"], 502) + self.assertIn("cut off", rec["error"]) + self.assertEqual(rec["reply"]["usage"]["input"], 7) + self.assertEqual(job._usage_total([rec])["cost"], rec["cost_usd"]) + + def test_a_refused_call_keeps_each_inline_image_by_its_type_wholeness_and_size_alone(self): + png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20 + b"IEND\xaeB`\x82\n").decode() + cut = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20).decode() + jpeg = base64.b64encode(b"\xff\xd8\xff" + b"y" * 9 + b"\xff\xd9tail").decode() + webp = base64.b64encode(b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"z" * 4 + b"\n").decode() + image = lambda kind, data: {"type": "image", "source": {"type": "base64", "media_type": kind, "data": data}} + content = [image("image/png", png), image("image/png", cut), image("image/gif", "not base64!"), + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg}"}}, + {"type": "input_image", "image_url": f"data:image/webp;base64,{webp}"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "grade"}] + sent = [{"type": "image/png", "is": "image/png", "bytes": len(png)}, {"type": "image/png", "is": None, "bytes": len(cut)}, + {"type": "image/gif", "is": None, "bytes": 11}, {"type": "image/jpeg", "is": "image/jpeg", "bytes": len(jpeg)}, + {"type": "image/webp", "is": "image/webp", "bytes": len(webp)}] + for mode in ("400", "413", "422", "429", None): + self.post("img", "/messages", {"model": "m", "mode": mode, "messages": [{"role": "user", "content": content}]}) + rec = self.relay.calls("img")[-1] + with self.subTest(mode): + self.assertEqual(rec.get("images"), sent if mode in ("400", "413", "422") else None) + self.assertFalse(any(data in json.dumps(rec) for data in (png, cut, jpeg, webp))) + + def test_a_200_to_a_model_call_with_no_reply_is_a_502_said_by_its_kind_alone(self): + for mode, path, code, said, kind in ( + ("hollow", "/messages", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/chat/completions", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/responses", 502, "no reply", "NetworkConnectionError"), + ("hollow-error", "/messages", 502, "an error 502", "NetworkConnectionError"), + ("hollow-429", "/messages", 429, "an error 429", "RateLimitError"), + ("hollow-404", "/completions", 502, "an error 404", "NetworkConnectionError"), + ("hollow-402", "/messages", 402, "an error 402", "ApiError"), + ("not-json", "/messages", 502, "no reply", "NetworkConnectionError")): + tag = f"uh-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + rec = self.relay.calls(tag)[-1] + self.assertEqual(self.retry_after, "30") + with self.subTest(mode=mode, path=path): + self.assertEqual((status, rec["status"]), (code, code)) + self.assertEqual(json.loads(body)["error"]["message"], f"the endpoint answered 200 with {said}") + self.assertEqual(rec["error"], f"HTTP {code}: the endpoint answered 200 with {said}") + self.assertEqual(rec["answered"], 200) + self.assertFalse({"reply", "cost_usd"} & set(rec)) + self.assertNotIn("the request said", json.dumps(rec)) + self.assertEqual(relay.failure([rec], agent=True)[0], kind) + self.assertEqual(self.post("uk-asked-stream", "/messages", {"model": "m", "mode": "hollow", "stream": True, + "messages": []}), (200, HOLLOW["hollow"])) + for mode, path in (("count", "/messages/count_tokens"), ("no-choice", "/chat/completions"), ("hollow", "/models"), + ("hollow", "/threads/t1/messages")): + tag = f"uk-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + with self.subTest(path): + self.assertEqual((status, body), (200, HOLLOW[mode])) + self.assertEqual(self.relay.calls(tag)[-1]["status"], 200) + self.assertNotIn("answered", self.relay.calls(tag)[-1]) + def test_streamed_responses_read_from_the_completed_event(self): status, body = self.post("u3", "/responses", {"model": "m", "stream": True, "input": "x"}) self.assertEqual(body, RESP_SSE) @@ -128,6 +226,25 @@ def test_streamed_responses_read_from_the_completed_event(self): self.assertEqual(rec["reply"]["tool_calls"][0]["name"], "shell") self.assertEqual(rec["reply"]["usage"]["cached"], 10) + def test_a_failed_response_with_no_usage_costs_nothing(self): + for tag, mode in (("uf1", "failed"), ("uf2", "failed-sse")): + status, _ = self.post(tag, "/responses", {"model": "m", "mode": mode, "input": "x", "stream": mode == "failed-sse"}) + self.assertEqual(status, 200) + rec = self.relay.calls(tag)[-1] + self.assertEqual((rec["reply"]["end"], rec["reply"]["error"]["code"]), ("failed", "server_error")) + self.assertEqual((rec["cost_usd"], rec["cost_from"]), (0.0, "failed")) + failed = {"status": 200, "cost_usd": 0.0, "cost_from": "failed"} + self.assertEqual(job._usage_total([failed, {"status": 200}])["cost"], None) + self.assertEqual({k: job._usage_total([failed, {"status": 200, "cost_usd": 0.5, "cost_from": "provider"}])[k] + for k in ("cost", "cost_from")}, {"cost": 0.5, "cost_from": "provider"}) + + def test_an_incomplete_response_is_priced_from_its_usage(self): + self.post("uf3", "/responses", {"model": "m", "mode": "incomplete", "input": "x"}) + rec = self.relay.calls("uf3")[-1] + self.assertTrue(rec["reply"]["end"].startswith("incomplete")) + self.assertEqual(rec["cost_from"], "estimate") + self.assertGreater(rec["cost_usd"], 0) + def test_calls_are_recorded_by_their_unit(self): self.post("ua", "/chat/completions", {"model": "m", "messages": []}) self.post("ub", "/chat/completions", {"model": "m", "messages": []}) @@ -230,6 +347,54 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_stream_cut_off_or_a_failed_response_with_no_usage_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + for tag, path, body in (("uc", "/chat/completions", {"model": "m", "stream": True, "mode": "cut", "messages": []}), + ("uf", "/responses", {"model": "m", "mode": "failed", "input": "x"})): + req = urllib.request.Request(r.url(tag) + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,), (failed,) = r.calls("uc"), r.calls("uf") + self.assertEqual((cut["status"], failed["cost_from"]), (502, "failed")) + self.assertIsNotNone(cut.get("cost_usd")) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], cut["booked_usd"] + failed["booked_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_stream_cut_off_that_already_cost_more_than_its_booking_counts_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "stream": True, "mode": "cut", "messages": []} + req = urllib.request.Request(r.url("uc") + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with mock.patch.object(ledger, "bound", lambda *a, **k: 1e-9): + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,) = r.calls("uc") + got = led.state() + self.assertEqual((cut["status"], got["unsettled_usd"]), (502, 0.0)) + self.assertGreater(cut["cost_usd"], cut["booked_usd"]) + self.assertAlmostEqual(got["spent_usd"], cut["cost_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_unit_asked_to_stop_books_nothing_and_a_200_with_no_reply_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + with open(os.path.join(r.dir, relay.STOP), "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(r, "u1", {"model": "m", "messages": []})[0], 400) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"], got["unsettled_usd"]), (0.0, 0, 0.0)) + os.unlink(os.path.join(r.dir, relay.STOP)) + self.assertEqual(self.post(r, "u1", {"model": "m", "mode": "hollow", "messages": []})[0], 502) + (call,) = r.calls("u1") + got = led.state() + self.assertEqual((call["answered"], got["spent_usd"], got["held_usd"]), (200, 0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], call["booked_usd"]) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply (2 output tokens here) a call that names no limit is booked those, at the same price.""" @@ -451,6 +616,11 @@ def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): self.assertEqual(relay._request_tail({"input": [], "reasoning": {"effort": "high"}})["reasoning_effort"], "high") self.assertIsNone(relay._request_tail({"messages": []})["reasoning_effort"]) + def test_reasoning_is_read_as_reasoning_or_thinking_tokens(self): + self.assertEqual(relay._usage({"completion_tokens_details": {"reasoning_tokens": 3}})["reasoning"], 3) + self.assertEqual(relay._usage({"output_tokens_details": {"reasoning_tokens": 5}})["reasoning"], 5) + self.assertEqual(relay._usage({"output_tokens": 9, "output_tokens_details": {"thinking_tokens": 7}})["reasoning"], 7) + class ServeTest(unittest.TestCase): """The relay as a process of its own: the key from its environment only, diff --git a/app/tests/test_standard_relay_pin.py b/app/tests/test_standard_relay_pin.py new file mode 100644 index 0000000..cfe525f --- /dev/null +++ b/app/tests/test_standard_relay_pin.py @@ -0,0 +1,263 @@ +"""A pinned route's failure on top of the serving pin (``serving``): a pinned call no endpoint took, or one its reply +says another provider served, fails the unit as its route's. Before a fake OpenRouter.""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import tempfile +import threading +import unittest +from unittest import mock +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from nullius.runs import classify +from nullius.standard import definitions, install, relay, run + +M3, SONNET = "minimax/minimax-m3", "anthropic/claude-sonnet-4.6" +PINS = {M3: ["minimax"], SONNET: ["anthropic"]} +SERVES = {M3: ["Parasail", "MiniMax"], SONNET: ["Google", "Anthropic"]} +SECRET = "the words of a gated task" +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") + + +class OpenRouter(BaseHTTPRequestHandler): + """Routes as OpenRouter documents them: ``down`` a 503, no endpoint of the pin a 404.""" + ignore_pin = False + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model, pref = body.get("model"), body.get("provider") or {} + allowed = [e for e in SERVES.get(model, ["Someone"]) + if OpenRouter.ignore_pin or not pref.get("order") or e.lower() in pref["order"]] + if body.get("down"): + allowed = [] + if not allowed: + status = 503 if body.get("down") else 404 + return self.reply(status, {"error": {"code": status, "message": f"No allowed providers are available for the selected model: {SECRET}"}}) + if self.path.endswith("/messages"): + return self.reply(200, {"id": "gen-555", "type": "message", "role": "assistant", "content": [{"type": "text", "text": SECRET}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) + self.reply(200, {"id": "gen-123", "provider": allowed[0], "choices": [{"message": {"role": "assistant", "content": SECRET}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": 0.0001}}) + + def reply(self, status, out): + body = json.dumps(out).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + try: + return e.code, e.read() + finally: + e.close() + + +class Pinned(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.relay = self.pinned(PINS) + + def pinned(self, serving, model=M3): + r = relay.Relay(self.base, "sk-real", tempfile.mkdtemp(dir=self.dir), model=model, + side_models=["claude-sonnet-4-6=" + SONNET], serving=serving).start() + self.addCleanup(r.stop) + return r + + def test_no_endpoint_of_the_pin_is_the_routes_error_not_an_empty_answer(self): + r = self.pinned({M3: ["fireworks"]}) + status, _ = post(r.url("u3") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertEqual(status, 404) + kind, said = relay.failure(r.calls("u3"), serving={M3: ["fireworks"]}) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to fireworks", said) + self.assertEqual(classify.from_verdict(kind, None), classify.API_ERROR) + + def test_a_pinned_provider_down_is_the_routes_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + (call,) = self.relay.calls("u1") + self.assertEqual(call["status"], 503) + self.assertEqual(relay.failure(self.relay.calls("u1"), serving=PINS)[0], "ApiProviderRouteError") + + def test_a_gated_benchmark_keeps_the_endpoints_words_out_of_a_route_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + kind, said = relay.failure(self.relay.calls("u1"), words=False, serving=PINS) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to minimax", said) + self.assertIn("answered 503", said) + self.assertNotIn(SECRET, said) + self.assertNotIn("No allowed providers", said) + + def test_an_unpinned_404_is_still_what_it_was(self): + calls = [{"status": 404, "model": "other/model", "error": "HTTP 404: model not found"}] + self.assertEqual(relay.failure(calls, serving=PINS)[0], "ModelNotFoundError") + self.assertIsNone(relay.route_failure(calls, PINS)) + + def test_a_side_models_refusal_fails_the_unit_even_when_the_agent_answered_last(self): + unit = self.relay.url("u1") + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + calls = self.relay.calls("u1") + self.assertIsNone(relay.failure(calls, serving=PINS)) + kind, said = relay.route_failure(calls, PINS, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn(SONNET, said) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": []}) + self.assertIsNone(relay.route_failure(self.relay.calls("u1"), PINS)) + + def test_a_routes_calls_are_read_in_the_order_they_began(self): + pin, refused = {M3: ["fireworks"]}, {"model": M3, "status": 404} + self.assertIsNone(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, dict(refused, t0=1)], pin)) + self.assertEqual(relay.route_failure([dict(refused, t0=2), {"model": M3, "status": 200, "t0": 1}], pin)[0], + "ApiProviderRouteError") + self.assertEqual(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, refused], pin)[0], "ApiProviderRouteError") + + def test_a_judges_answer_does_not_cover_the_simulated_users_refusal_of_the_same_model(self): + unit, judge = self.relay.url("u1"), self.relay.url("u1-judge", model=SONNET) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + post(judge + "/messages", {"model": SONNET, "messages": []}) + routes = [self.relay.calls("u1"), self.relay.calls("u1-judge"), []] + self.assertEqual(relay.routes_failure(routes, PINS, words=False)[0], "ApiProviderRouteError") + self.assertIsNone(relay.routes_failure([self.relay.calls("u1-judge")], PINS)) + + def test_only_a_route_with_no_endpoint_stops_the_run(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + self.assertFalse(relay.route_dead(relay.failure(self.relay.calls("u1"), serving=PINS))) + r = self.pinned({M3: ["fireworks"]}) + post(r.url("u5") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertTrue(relay.route_dead(relay.failure(r.calls("u5"), words=False, serving={M3: ["fireworks"]}))) + self.assertFalse(relay.route_dead(("ApiError", "404: x"))) + + def test_an_answer_from_outside_the_pin_fails_the_unit_as_its_route(self): + pin = {"openai/gpt-5-mini": ["minimax"]} + r = self.pinned(pin, model="openai/gpt-5-mini") + SERVES["openai/gpt-5-mini"] = ["Parasail"] + self.addCleanup(SERVES.pop, "openai/gpt-5-mini") + OpenRouter.ignore_pin = True + self.addCleanup(setattr, OpenRouter, "ignore_pin", False) + post(r.url("u4") + "/chat/completions", {"model": "x", "messages": []}) + (call,) = r.calls("u4") + self.assertEqual((call["status"], call["reply"]["served_by"]), (200, "Parasail")) + self.assertIsNone(relay.failure(r.calls("u4"), serving=pin)) + kind, said = relay.route_failure(r.calls("u4"), pin, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("served by Parasail", said) + self.assertNotIn(SECRET, said) + self.assertIsNone(relay.route_failure(r.calls("u4"), {})) + + def test_a_provider_is_off_its_pin_only_when_its_name_shares_nothing_with_the_pin(self): + self.assertIs(relay.held("MiniMax", ["minimax"]), True) + self.assertIs(relay.held("Anthropic", ["anthropic"]), True) + self.assertIs(relay.held("Google AI Studio", ["google-ai-studio"]), True) + self.assertIs(relay.held("Parasail", ["minimax"]), False) + self.assertIsNone(relay.held("Google", ["google-vertex"])) + self.assertIsNone(relay.route_failure([{"status": 200, "model": "m", "reply": {"served_by": "Google"}}], {"m": ["google-vertex"]})) + + def test_a_name_that_is_not_a_providers_name_fails_nothing_and_is_never_said(self): + odd = [{"status": 200, "model": "m", "reply": {"served_by": "Parasail\n" + SECRET}}] + self.assertIsNone(relay.route_failure(odd, {"m": ["minimax"]}, words=False)) + + +class Definition(unittest.TestCase): + def test_a_benchmark_without_pins_runs_as_before(self): + benches, _ = definitions.load("benchmark") + for b in benches: + with self.subTest(b["key"]): + self.assertEqual(run.pins_of(b, {"_kind": "agent"}, "openrouter", "m"), {}) + for which in (b.get("setups") or {}): + if which == "original": + other, _ = definitions.setup(b, which, "", "openrouter") + self.assertNotIn("pins", other or {}) + + def test_the_definitions_pin_holds_whatever_served_by_says(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + make = lambda **kw: run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", + env={"OPENROUTER_API_KEY": "k"}, unsandboxed=True, jobs_root=tmp, say=lambda s: None, **kw) + self.assertEqual(make().serving, {SONNET: ["anthropic"]}) + r = make(served_by={"m-1": ["prov-a"], SONNET: ["google-vertex"]}) + self.assertEqual((r.serving, r.route_pins), ({"m-1": ["prov-a"], SONNET: ["anthropic"]}, {SONNET: ["anthropic"]})) + + +class Wired(unittest.TestCase): + """A unit of a run whose judge the definition pins and whose own model the user pins (``--served-by``).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(self.tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + self.r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "k"}, + unsandboxed=True, jobs_root=self.tmp, say=lambda s: None, served_by={"m-1": ["prov-a"]}) + + def ended(self, routes): + rr = mock.Mock(mcp_listeners=[], unit_image="", dir=self.tmp) + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 0} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit"): + stack.enter_context(mock.patch.object(self.r, name)) + self.noted = stack.enter_context(mock.patch.object(self.r, "note_refusal")) + stack.enter_context(mock.patch.object(self.r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(self.r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(self.r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = self.r.unit(0, "good") + return (res.get("exception_info") or {}).get("exception_type") + + def test_only_a_route_the_definition_pins_fails_the_unit_as_its_route(self): + ok, no = {"model": "m-1", "status": 200}, {"status": 404, "error": "HTTP 404: no endpoint"} + self.assertNotIn(self.ended({"u0000": [dict(no, model="m-1")]}), (None, "ApiProviderRouteError")) + self.assertIsNone(self.ended({"u0000": [dict(ok, reply={"served_by": "Parasail"})]})) + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertTrue(self.noted.call_args[0][0][1].startswith("404: ")) + self.assertEqual(self.ended({"u0000": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [{"model": SONNET, "status": 200, "reply": {"served_by": "Parasail"}}]}), + "ApiProviderRouteError") + + def test_three_units_in_a_row_with_no_endpoint_on_a_pinned_route_stop_the_run(self): + self.r._halt, self.r._refused, self.r.halted = threading.Event(), [], None + for _ in range(3): + self.r.note_refusal(("ApiProviderRouteError", "404: no endpoint")) + self.assertTrue(self.r._halt.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec26..4defc5c 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -624,6 +624,35 @@ def unit(*a): got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_benchmark_that_says_so_runs_no_rate_limited_unit_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}], + limits={"run_again": False}) + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + real = r.unit + + def unit(*a): + res = real(*a) + if res and res["trial_name"].startswith("model__"): + with open(os.path.join(r.job_dir, res["trial_name"], ".work", "relay", relay.STOP), "w") as f: + f.write("batch cap\n") + return res + r.unit = unit + job_dir = r.start() + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py new file mode 100644 index 0000000..fe861a2 --- /dev/null +++ b/app/tests/test_standard_served_by.py @@ -0,0 +1,768 @@ +"""A pin of the serving providers that may answer a model (``--served-by``): +the relay puts OpenRouter's routing field on every request it sends as that +model and on no other, in place of the caller's own; each call records the +routing sent, the provider that served the reply, how the reply ended and an +error inside a success, as cleaned copies, read apart from the rest of the +record; the job and each trial say who served the calls of every route and +whether the pin held; ``merge`` refuses shards of different pins; a +container's relay is given the pin as the unit's own relay is. A run without +the option sends the same requests and ends its units as it did. Every +upstream here is a fake on this machine's loopback: no request leaves it.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import select +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +import urllib.request +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import progress +from nullius.standard import cli, container, definitions, install, job, merge, relay, run + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIN = {"order": ["prov-a", "prov-b/fp8"], "allow_fallbacks": False} +KEY = "or-test-relay-key-0123456789" +NEW = ("served_by", "end", "error") # what the record's reply gains, beside what it held + + +def sse(*events) -> bytes: + return b"".join(b"data: " + json.dumps(e).encode() + b"\n\n" for e in events) + b"data: [DONE]\n\n" + + +def routing_facts(selected, *others) -> dict: + """OpenRouter's routing facts (``openrouter_metadata``) as a reply holds them when asked: the endpoint that + served it is the one marked ``selected`` among the available ones.""" + available = [{"provider": o, "model": "asked", "selected": False} for o in others] + if selected is not None: + available.insert(len(available) // 2, {"provider": selected, "model": "asked", "selected": True}) + return {"requested": "asked", "attempt": 1, "summary": f"available={len(available)}, selected={selected}", + "endpoints": {"total": len(available) + 2, "available": available}} + + +# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions and Messages of +# themselves; Responses in the routing facts alone, given when the request asks for them (``ASKED``). +CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}} +CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "h"}, "finish_reason": None}]}, + {"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "i"}, "finish_reason": "length"}]}, + {"id": "c1", "provider": "Prov A", "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1}}) +MESSAGES = {"type": "message", "provider": "Prov B", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}} +MESSAGES_SSE = sse({"type": "message_start", "message": {"type": "message", "provider": "Prov B", "usage": {"input_tokens": 3}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 1}}, + {"type": "message_stop"}) +RESPONSES = {"object": "response", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}], + "usage": {"input_tokens": 3, "output_tokens": 1}} +COMPLETED = {"object": "response", "status": "completed", "output": [], "usage": {"input_tokens": 3, "output_tokens": 1}} +RESPONSES_SSE = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": COMPLETED}) +RESPONSES_ASKED = dict(RESPONSES, openrouter_metadata=routing_facts("Prov C", "Prov D", "Prov E")) +RESPONSES_SSE_ASKED = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov C", "Prov D"))}) +UNASKED = {False: RESPONSES, True: RESPONSES_SSE} +SHAPES = {("/chat/completions", False): (CHAT, "Prov A", "stop"), ("/chat/completions", True): (CHAT_SSE, "Prov A", "length"), + ("/messages", False): (MESSAGES, "Prov B", "end_turn"), ("/messages", True): (MESSAGES_SSE, "Prov B", "max_tokens"), + ("/responses", False): (RESPONSES_ASKED, "Prov C", "incomplete: max_output_tokens"), + ("/responses", True): (RESPONSES_SSE_ASKED, "Prov C", "completed")} + + +def body_for(path: str, stream: bool) -> dict: + """A caller's request of that shape.""" + if path == "/responses": + return {"model": "asked", "input": "hello", "stream": stream} + return {"model": "asked", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5, "stream": stream} + + +def without_new(reply: dict) -> dict: + return {k: v for k, v in reply.items() if k not in NEW} + + +class Upstream(BaseHTTPRequestHandler): + """A provider on the loopback: keeps every request as it came, bytes and headers, and answers it in its shape; + ``Upstream.next`` replaces the next reply (status, content type, body), ``Upstream.always`` every one.""" + seen = [] + next = None + always = None + + def log_message(self, *a): + pass + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + Upstream.seen.append((self.path, {k.lower(): v for k, v in self.headers.items()}, raw)) + req = json.loads(raw) + if Upstream.next is not None or Upstream.always is not None: + (status, kind, body), Upstream.next = Upstream.next or Upstream.always, None + else: + path = "/" + self.path.split("/v1/", 1)[-1] + reply = SHAPES.get((path, bool(req.get("stream"))), (CHAT,))[0] + if path == "/responses" and self.headers.get("X-OpenRouter-Metadata") != "enabled": + reply = UNASKED[bool(req.get("stream"))] + status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ + reply if isinstance(reply, bytes) else json.dumps(reply).encode() + self.send_response(status) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url: str, body: dict, headers=None) -> bytes: + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY, + **(headers or {})}) + with urllib.request.urlopen(req, timeout=10) as r: + return r.read() + + +class WithUpstream(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.tmp = tempfile.mkdtemp() + Upstream.seen, Upstream.next, Upstream.always = [], None, None + self.relays = [] + self.addCleanup(setattr, Upstream, "always", None) + + def tearDown(self): + for r in self.relays: + r.stop() + shutil.rmtree(self.tmp, ignore_errors=True) + + def relay(self, **kw): + r = relay.Relay(self.base, kw.pop("key", "or-test"), os.path.join(self.tmp, f"rec{len(self.relays)}"), **kw).start() + self.relays.append(r) + return r + + def sent(self, i=-1) -> dict: + return json.loads(Upstream.seen[i][2]) + + +class TheRelay(WithUpstream): + def test_the_pin_goes_on_every_request_sent_as_the_pinned_model_and_on_no_other(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0-judge", model="m-judge") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0") + "/messages", body_for("/messages", True)) + self.assertEqual([(json.loads(raw)["model"], json.loads(raw).get("provider")) for _, _, raw in Upstream.seen], + [("m-run", PIN), ("m-judge", None), ("m-run", PIN)]) + self.assertEqual([c["request"].get("routing") for c in r.calls("u0")], [PIN, PIN]) + self.assertNotIn("routing", r.calls("u0-judge")[0]["request"]) + + def test_a_callers_own_routing_is_replaced_and_both_are_recorded(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"sort": "price"})) + self.assertEqual(self.sent()["provider"], PIN) + (c,) = r.calls("u0") + self.assertEqual((c["request"]["routing"], c["request"]["routing_asked"]), (PIN, {"sort": "price"})) + + def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self): + r = self.relay(model="m-run") + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"only": ["prov-z"]})) + self.assertEqual(self.sent()["provider"], {"only": ["prov-z"]}) + (c,) = r.calls("u0") + self.assertEqual(c["request"]["routing"], {"only": ["prov-z"]}) + self.assertNotIn("routing_asked", c["request"]) + + def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): + """An error that echoes the request's credentials, and a routing object holding a token: the record keeps + neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" + Upstream.always = (200, "application/json", json.dumps({"choices": [{"message": {"content": "ok"}}], + "error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + for serving in (None, {"m-run": ["prov-a"]}): + with self.subTest(serving=serving): + r = self.relay(model="m-run", key=KEY, serving=serving) + asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}", KEY: "as a key"}) + post(r.url("u0") + "/chat/completions", asked) + c = r.calls("u0")[-1] + self.assertNotIn(KEY, json.dumps(c)) + self.assertEqual(c["reply"]["error"]["code"], 401) + if serving is None: # sent as the caller sent it: the record alone is cleaned + self.assertEqual(self.sent()["provider"], asked["provider"]) + self.assertEqual(c["request"]["routing"]["token"], "[REDACTED]") + else: + self.assertEqual(c["request"]["routing_asked"]["token"], "[REDACTED]") + + def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + for (path, stream), (_, who, end) in SHAPES.items(): + with self.subTest(path=path, stream=stream): + post(r.url("u0") + path, body_for(path, stream)) + self.assertEqual(self.sent()["provider"], {"order": ["prov-a"], "allow_fallbacks": False}) + reply = r.calls("u0")[-1]["reply"] + self.assertEqual((reply.get("served_by"), reply.get("end"), reply.get("error")), (who, end, None)) + # what was read before is read as before, beside the new facts + self.assertEqual(without_new(reply), relay.read_reply(path, SHAPES[(path, stream)][0] if stream + else json.dumps(SHAPES[(path, stream)][0]).encode(), stream)) + + def test_several_choices_keep_their_endings_and_errors_by_index(self): + body = {"choices": [{"index": 0, "message": {"content": "ok"}, "finish_reason": "stop"}, + {"index": 1, "message": {}, "finish_reason": "error", "error": {"code": 429, "message": "limited"}}]} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(body).encode(), False), + {"end": {"0": "stop", "1": "error"}, "error": {"code": 429, "message": "limited", "choice": 1}}) + stream = sse({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}, {"index": 1, "delta": {}, "finish_reason": None}]}, + {"choices": [{"index": 1, "delta": {}, "finish_reason": "stop"}]}) + self.assertEqual(relay.reply_facts("/chat/completions", stream, True)["end"], {"0": "length", "1": "stop"}) + + def test_an_error_beside_a_whole_reply_is_read_in_every_shape(self): + error = {"code": 502, "message": "failed"} + for path, body in (("/messages", {"type": "message", "content": [], "error": error}), + ("/embeddings", {"object": "list", "data": [], "error": error}), + ("/chat/completions", {"choices": [{"message": {"content": "ok"}}], "error": error}), + ("/responses", {"object": "response", "status": "failed", "output": [], "error": error})): + with self.subTest(path=path): + self.assertEqual(relay.reply_facts(path, json.dumps(body).encode(), False)["error"], error) + + def test_a_completed_responses_events_routing_facts_name_who_served_it(self): + body = sse({"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov A", "Prov X"))}) + self.assertEqual(relay.reply_facts("/responses", body, True), {"served_by": "Prov A", "end": "completed"}) + whole = json.dumps(dict(RESPONSES, openrouter_metadata=routing_facts("Prov A", "Prov X", "Prov Y"))).encode() + self.assertEqual(relay.reply_facts("/responses", whole, False)["served_by"], "Prov A") + + def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): + odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"endpoints": {"available": 1}}}, + {"type": "message", "content": [], "provider": 7, "error": "a string", "stop_reason": ["x"]}, + {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}, + {"choices": [], "openrouter_metadata": {"endpoints": [1]}}, + {"choices": [], "openrouter_metadata": {"endpoints": {"available": [7, {"provider": 7, "selected": True}, + {"provider": "Prov A", "selected": "true"}]}}}] + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + for body in odd: + with self.subTest(body=body): + raw = json.dumps(body).encode() + Upstream.next = (200, "application/json", raw) + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)), raw) + self.assertEqual({k: v for k, v in r.calls("u0")[-1]["reply"].items() if k in NEW}, {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"[1, 2]", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) + + def test_a_failed_responses_stream_is_read_beside_its_facts(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage are read from its + ``response.failed`` event, its error beside them.""" + body = sse({"type": "response.output_text.delta", "delta": "part"}, + {"type": "response.failed", "response": {"object": "response", "status": "failed", + "error": {"code": "server_error", "message": "failed"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], + "usage": {"input_tokens": 5, "output_tokens": 2}}}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "x", "tool_calls": [], "usage": relay._usage({"input_tokens": 5, "output_tokens": 2})}) + self.assertEqual(relay.reply_facts("/responses", body, True), + {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) + + def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): + """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own + is added, and a unit's failure reads it as before (a 200 is no failure).""" + r = self.relay(model="m-run") + errors = [(sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, + "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", + {"code": "server_error", "message": "upstream ended"}), + (sse({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), "text/event-stream", + {"type": "overloaded_error", "message": "Overloaded"})] + for body, kind, said in errors: + with self.subTest(said=said): + Upstream.next = (200, kind, body) + streamed = kind == "text/event-stream" + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", streamed)), body) + c = r.calls("u0")[-1] + self.assertEqual((c["status"], c["reply"]["error"]), (200, said)) + self.assertNotIn("error", c) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", body, streamed)) + self.assertIsNone(relay.failure(r.calls("u0"))) + + def test_the_routing_facts_are_asked_of_openrouter_alone_and_always_as_enabled(self): + meta = json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, + openrouter_metadata=routing_facts("Prov A", "Prov X"))).encode() + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + Upstream.next = (200, "application/json", meta) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False), {"X-OpenRouter-Metadata": "disabled"}) + got = [v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"] + self.assertEqual(got, ["enabled"]) + self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") + for stream in (False, True): # a Responses request: asked too + post(r.url("u0") + "/responses", body_for("/responses", stream), {"X-OpenRouter-Metadata": "disabled"}) + self.assertEqual([v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"], ["enabled"]) + post(r.url("u0") + "/messages", body_for("/messages", False)) # a Messages reply names its provider: nothing asked + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + for kw in ({"serving": {"m-run": ["prov-a"]}}, {"facts": True}): # another provider's address; no pin + with self.subTest(kw=kw): + other = self.relay(model="m-run", **kw) + post(other.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + self.assertTrue(relay.Relay("https://openrouter.ai/api/v1", "k", os.path.join(self.tmp, "or")).facts) + self.assertFalse(relay.Relay("https://api.openai.com/v1", "k", os.path.join(self.tmp, "oa")).facts) + + def test_without_a_pin_every_request_goes_on_byte_for_byte_as_before(self): + """What a relay sent before pins existed: the caller's body with the model set (and a streamed Chat + Completions request's usage flag), serialised once, and the caller's headers but its credentials. The + three shapes, streamed or not, with and without a routing field of the caller's, as the run's model, a + side model and a judge's route's model; with no pin, or a pin on another model.""" + for serving in (None, {"m-other": ["prov-a"]}): + r = self.relay(model="m-run", side_models=("m-side",), serving=serving) + for (path, stream) in SHAPES: + for extra in ({}, {"provider": {"order": ["prov-z"]}}): + for url, asked, model in ((r.url("u0"), "asked", "m-run"), (r.url("u0"), "m-side", "m-side"), + (r.url("u0-judge", model="m-judge"), "asked", "m-judge")): + with self.subTest(serving=serving, path=path, stream=stream, extra=extra, model=model): + req = dict(body_for(path, stream), model=asked, **extra) + post(url + path, req, {"X-Caller": "kept"}) + expected = dict(req, model=model) + if stream and path == "/chat/completions": + expected.setdefault("stream_options", {})["include_usage"] = True + sent_path, headers, raw = Upstream.seen[-1] + self.assertEqual(raw, json.dumps(expected).encode()) + self.assertEqual(sent_path, "/v1" + path) + self.assertEqual({k: v for k, v in headers.items() if k not in ("host", "content-length")}, + {"content-type": "application/json", "x-caller": "kept", "accept": "*/*", + "accept-encoding": "identity", "authorization": "Bearer or-test", + "user-agent": headers["user-agent"]}) + + +class TheSidecar(WithUpstream): + def setUp(self): + super().setUp() + self.enterContext(mock.patch.object(container, "_security", "")) # a default rootful engine, never asked + + def test_the_containers_relay_is_given_the_pin_on_its_command_line(self): + argv = container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", base="http://x/v1", + model="m-run", routes={"u0-judge": "m-judge"}, + serving={"m-run": ["prov-a", "prov-b/fp8"], "m-judge": ["prov-c"]}) + got = [argv[i + 1] for i, a in enumerate(argv) if a == "--served-by"] + self.assertEqual(got, ["m-judge=prov-c", "m-run=prov-a,prov-b/fp8"]) + self.assertNotIn("--served-by", container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", + base="http://x/v1", model="m-run", routes={})) + + def test_the_unit_hands_its_pin_to_the_sidecars_command(self): + unit = container.Unit("job", "u0", os.path.join(self.tmp, "rec"), image="img", app=APP, base=self.base, key="or-test", + model="m-run", serving={"m-run": ["prov-a"]}) + ran = [] + + def docker(args, **kw): + ran.append(args) + if "run" in args[:1]: + raise container.Unavailable("no engine in this test") + return subprocess.CompletedProcess(args, 0, "", "") + with mock.patch.object(container, "_docker", docker), self.assertRaises(container.Unavailable): + unit.start() + (started,) = [a for a in ran if a[:1] == ["run"]] + self.assertIn("m-run=prov-a", started[started.index("--served-by") + 1:]) + + def test_the_relays_own_command_refuses_a_pin_it_cannot_take(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + for given in (["m=bad slug"], ["m=prov-a,prov-a"], ["m="], ["m=prov-a", "m=prov-b"], ["prov-a"]): + with self.subTest(given=given): + # a process of its own, bounded: a relay that took the pin would serve until stopped + got = subprocess.run([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", + "--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, + *[w for g in given for w in ("--served-by", g)]], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), capture_output=True, text=True, timeout=20) + self.assertEqual(got.returncode, 2, got.stdout) + self.assertIn("error", json.loads(got.stdout.splitlines()[0])) + + def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + p = subprocess.Popen([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", "--base", self.base, + "--record", os.path.join(self.tmp, "rec"), "--key-file", key, "--model", "m-run", + "--served-by", "m-run=prov-a,prov-b/fp8", "--routing"], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) + try: + ready, _, _ = select.select([p.stdout], [], [], 20) # its one line, or no more than 20 s + self.assertTrue(ready, "the relay did not say it listens within 20 s") + url = json.loads(p.stdout.readline())["listening"] + post(url + "/u0/v1/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(self.sent()["provider"], PIN) + finally: + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + p.wait(10) + p.stdout.close() + + +class TheRun(WithUpstream): + def setUp(self): + super().setUp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = definitions.get("benchmark", "fakebench", ROOT) + self.a = definitions.get("agent", "fake", ROOT) + + def tearDown(self): + install.HOME = self._home + super().tearDown() + + def make(self, **kw): + kw.setdefault("other", self.a) + return run.Run(kw.pop("bench", self.b), kw.pop("other"), model=kw.pop("model", "m-1"), provider=kw.pop("provider", "openrouter"), + env=kw.pop("env", {"OPENROUTER_API_KEY": "or-test"}), unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, **kw) + + def test_the_option_reads_model_and_slugs_and_refuses_what_it_cannot_take(self): + self.assertEqual(run.served_by(["prov-a", "m-judge=prov-b/fp8, prov-c"], "m-run"), + {"m-run": ["prov-a"], "m-judge": ["prov-b/fp8", "prov-c"]}) + for given, said in ((["m-run=prov-a,"], "not a serving provider's slug"), (["m-run=a b"], "not a serving provider's slug"), + (["prov-a,prov-a"], "named twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), + (["m-run="], "not a serving provider's slug")): + with self.subTest(given=given), self.assertRaisesRegex(run.Refused, said): + run.served_by(given, "m-run") + with self.assertRaisesRegex(run.Refused, "MODEL=SLUG"): + run.served_by(["prov-a"], "") + + def test_a_run_refuses_a_pin_it_cannot_take_however_it_is_given(self): + for pin in ({"m-1": []}, {"m-1": ["bad slug"]}, {"m-1": ["prov-a", "prov-a"]}, {"": ["prov-a"]}, {"m-1": "prov-a"}): + with self.subTest(pin=pin), self.assertRaisesRegex(run.Refused, "--served-by"): + self.make(served_by=pin) + + def test_a_provider_that_routes_no_model_to_several_is_refused_before_any_unit(self): + for provider, env in (("openai", {"OPENAI_API_KEY": "sk-x"}), ("anthropic", {"ANTHROPIC_API_KEY": "x"}), + ("custom", {"MODEL_API_URL": self.base})): + with self.subTest(provider=provider), self.assertRaisesRegex(run.Refused, "no serving providers to choose among"): + self.make(provider=provider, env=env, served_by={"m-1": ["prov-a"]}) + + def test_a_model_the_run_does_not_call_is_refused_before_any_unit(self): + with self.assertRaisesRegex(run.Refused, "names m-other, which this run does not call: it calls m-1"): + self.make(served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + nop = definitions.control(dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), "nop") + with self.assertRaisesRegex(run.Refused, "names m-1, which this run does not call: it calls no model"): + self.make(other=nop, bench=dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), + served_by={"m-1": ["prov-a"]}) + + def test_the_command_line_refuses_each_with_exit_2(self): + for patch in (mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_other", return_value=self.a), + mock.patch.object(cli, "_exit_on_term"), mock.patch.object(cli.settings, "load"), + mock.patch.object(progress, "ECHO", None)): + patch.start() + self.addCleanup(patch.stop) + for extra, said in ((["--provider", "openai", "--served-by", "prov-a"], "no serving providers"), + (["--served-by", "m-2=prov-a"], "does not call"), + (["--served-by", "m-1=bad slug"], "not a serving provider's slug"), + (["--served-by", "m-1="], "not a serving provider's slug"), + (["--served-by", "prov-a,prov-a"], "named twice"), + (["--served-by", "prov-a", "--served-by", "m-1=prov-b"], "m-1 twice")): + with self.subTest(extra=extra), mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-x", "OPENROUTER_API_KEY": "or-test"}): + out = io.StringIO() + with redirect_stdout(out): + code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--provider", "openrouter", + "--jobs", os.path.join(self.tmp, "jobs"), *extra]) + self.assertEqual(code, 2) + self.assertTrue(out.getvalue().strip().splitlines()[-1].startswith("refused:"), out.getvalue()) + self.assertIn(said, out.getvalue()) + + def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + with mock.patch.dict(run.PROVIDERS, {"openrouter": dict(run.PROVIDERS["openrouter"], base=self.base)}): + job_dir = run.run(b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding", + served_by={"m-1": ["prov-a"]}) + self.assertTrue(Upstream.seen) + self.assertTrue(all(json.loads(raw).get("provider") == {"order": ["prov-a"], "allow_fallbacks": False} + for _, _, raw in Upstream.seen)) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + trials = {} + for d in os.listdir(job_dir): + if "__u" in d: + with open(os.path.join(job_dir, d, "result.json")) as f: + trials[d.split("__")[0]] = json.load(f)["standard"] + n = len(Upstream.seen) + self.assertEqual(trials["model"]["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + self.assertEqual(trials["good"]["served"], {"calls": {}, "unnamed": 0, "held": {"m-1": None}}) # it called no model + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + self.assertTrue(all(c["request"]["routing"]["order"] == ["prov-a"] and c["reply"]["served_by"] == "Prov A" for c in calls)) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + + def test_a_run_without_the_option_sends_records_and_ends_as_before(self): + """Every request as before, the record's earlier fields as they were read, and the same verdicts, even + when the provider answers a 200 that holds an error: the relay reads that error beside the reply and the + unit ends as it did.""" + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + reply = dict({k: v for k, v in CHAT.items() if k != "provider"}, error={"code": 502, "message": "Provider returned error"}) + Upstream.always = (200, "application/json", json.dumps(reply).encode()) + job_dir = run.run(b, self.a, model="m-1", provider="custom", env={"MODEL_API_URL": self.base, "MODEL_API_KEY": "k"}, + unsandboxed=True, jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding") + self.assertTrue(Upstream.seen) + for _, headers, raw in Upstream.seen: + self.assertNotIn("provider", json.loads(raw)) + self.assertNotIn("x-openrouter-metadata", headers) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("served_by", json.load(f)["standard"]) + with open(os.path.join(job_dir, "model__u0002", "result.json")) as f: + res = json.load(f) + self.assertNotIn("exception_info", res) # a 200 is no failure: the unit's verdict stands + self.assertEqual(res["verifier_result"]["rewards"]["reward"], 0.0) + self.assertNotIn("served", res["standard"]) # no call answered, no pin: nothing to say + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + for c in calls: + self.assertEqual((c["status"], "error" in c), (200, False)) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", json.dumps(reply).encode(), False)) + self.assertEqual(c["reply"]["error"], {"code": 502, "message": "Provider returned error"}) + self.assertNotIn("routing", c["request"]) + + def test_the_summary_counts_every_route_of_the_unit(self): + """A judge on the pinned model, served by another provider than the agent's: the pin does not hold.""" + r = self.make(judge_model="m-1", served_by={"m-1": ["prov-a"]}) + call = lambda who: {"model": "m-1", "status": 200, "reply": {"served_by": who}} + routes = {"u0000": [call("Prov A")], "u0000-judge": [call("Prov Z")], "u0000-simulator": [call("Prov A")]} + rr = mock.Mock(mcp_listeners=[], unit_image="") + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 1} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit", "note_refusal"): + stack.enter_context(mock.patch.object(r, name)) + stack.enter_context(mock.patch.object(r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = r.unit(0, "good") + self.assertEqual(res["standard"]["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + self.assertEqual(res["agent_result"]["n_model_calls"], 1) # the tokens and cost: the agent's calls, as before + + def test_the_benchmarks_own_relay_counts_in_the_job(self): + r = self.make(served_by={"m-1": ["prov-a"]}) + folder = os.path.join(r.job_dir, ".work", "relay-benchmark") + os.makedirs(folder) + with open(os.path.join(folder, "embed-1.jsonl"), "w") as f: + f.write(json.dumps({"model": "m-1", "status": 200, "reply": {"served_by": "Prov Z"}}) + "\n") + bench = job.served(r.benchmark_calls(), r.serving) + job.write_result(r.job_dir, r.name, 0, 1, [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 0, "held": {"m-1": True}}}}], + 1, served=bench) + with open(os.path.join(r.job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + def test_the_containers_unit_is_given_the_runs_pin(self): + with mock.patch.object(container, "unusable", return_value=None): + r = run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, backend="container", + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + r.image, r.setup_driver, r.licences = "img", None, run.Licences(self.b, given={}) + trial = os.path.join(self.tmp, "trial") + with mock.patch.object(container, "Unit") as unit, mock.patch.object(run.Run, "unit_image", return_value="img"): + r.unit_relay("u0000", trial, {}) + self.assertEqual(unit.call_args.kwargs["serving"], {"m-1": ["prov-a"]}) + + +class TheRecord(unittest.TestCase): + def call(self, model, who=None, status=200, error=None): + reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} + return {"model": model, "status": status, "reply": reply} + + def test_who_served_a_units_calls_and_whether_the_pin_held(self): + pin = {"m-1": ["google-ai-studio", "prov-b/fp8"], "m-judge": ["prov-c"]} + self.assertEqual(job.served([self.call("m-1", "Google AI Studio"), self.call("m-1", "Prov B"), self.call("m-1", status=429)], pin), + {"calls": {"Google AI Studio": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": True, "m-judge": None}}) + self.assertEqual(job.served([self.call("m-1", "Prov Z"), self.call("m-1")], pin)["held"]["m-1"], "unknown") + self.assertEqual(job.served([self.call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered + self.assertEqual(job.served([self.call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) + self.assertIsNone(job.served([self.call("m-1", status=429)], None)) + + def test_a_name_reads_as_a_slug_by_its_case_and_spaces_alone(self): + pin = {"m-1": ["prov-a"]} + for name, held in (("Prov A", True), ("prov-a", True), ("PROV A", True), ("Prov/A", "unknown"), ("Prov_A", "unknown"), + ("Prov.A", "unknown"), ("Prov Ab", "unknown")): + with self.subTest(name=name): + self.assertEqual(job.served([self.call("m-1", name)], pin)["held"]["m-1"], held) + + def test_answered_calls_that_name_no_provider_are_counted(self): + known, unknown = job.served([self.call("m-1", "Prov A")]), job.served([self.call("m-1")]) + self.assertEqual(unknown, {"calls": {}, "unnamed": 1}) + self.assertEqual(job.served_total([{"standard": {"served": known}}, {"standard": {"served": unknown}}, {"standard": {}}]), + {"calls": {"Prov A": 1}, "unnamed": 1}) + parts = [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 1, "held": {"m-1": True}}}}, + {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}] + self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) + + +class TheReview(unittest.TestCase): + """What a review of the pin found, each a rule of its own.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_the_selected_endpoint_of_the_replys_end_is_recorded_never_another(self): + """The routing facts list the endpoints available: the one marked selected served the reply, never the + first listed nor one an earlier event's facts marked.""" + events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov B", "Prov A", "Prov C"))}] + facts = relay.reply_facts("/responses", sse(*events), True) + held = job.served([{"status": 200, "model": "m", "reply": facts}], {"m": ["prov-a"]})["held"]["m"] + self.assertEqual((facts["served_by"], held), ("Prov B", "unknown")) + body = dict(RESPONSES, openrouter_metadata=routing_facts("Prov B", "Prov A")) + self.assertEqual(relay.reply_facts("/responses", json.dumps(body).encode(), False)["served_by"], "Prov B") + + def test_facts_that_mark_no_single_named_endpoint_take_no_earlier_name(self): + both = routing_facts("Prov A", "Prov X") + for e in both["endpoints"]["available"]: + e["selected"] = True + for meta in (routing_facts(None, "Prov A", "Prov X"), routing_facts(7, "Prov A"), both): + with self.subTest(meta=meta): + body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": meta} + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) + stream = sse({"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=meta)}) + self.assertNotIn("served_by", relay.reply_facts("/responses", stream, True)) + chunks = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "openrouter_metadata": meta}) + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", chunks, True)) + + def test_a_pinned_responses_call_is_recorded_with_its_provider_and_the_pin_holds(self): + """A Responses reply names its provider in the routing facts alone: the relay asks for them on a pinned + request to OpenRouter, records the endpoint they mark, and the pin reads held, streamed or not. Without + them the reply names no provider and the pin reads unknown.""" + up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=up.serve_forever, daemon=True).start() + self.addCleanup(up.server_close) + self.addCleanup(up.shutdown) + Upstream.seen, Upstream.next, Upstream.always = [], None, None + base, pin = f"http://127.0.0.1:{up.server_address[1]}/v1", {"m-run": ["prov-c"]} + for facts, who, held in ((True, "Prov C", True), (False, None, "unknown")): + r = relay.Relay(base, "or-test", os.path.join(self.tmp, f"rec-{facts}"), model="m-run", serving=pin, facts=facts).start() + self.addCleanup(r.stop) + for stream in (False, True): + with self.subTest(facts=facts, stream=stream): + post(r.url("u0") + "/responses", body_for("/responses", stream)) + self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled" if facts else None) + self.assertEqual(json.loads(Upstream.seen[-1][2])["provider"], {"order": ["prov-c"], "allow_fallbacks": False}) + self.assertEqual(r.calls("u0")[-1]["reply"].get("served_by"), who) + self.assertEqual(job.served(r.calls("u0"), pin)["held"], {"m-run": held}) + + def test_an_error_keeps_its_stated_fields_alone(self): + nested = relay.reply_facts("/chat/completions", json.dumps({"error": {"api_key": ["review-only-credential"]}}).encode(), False) + self.assertNotIn("review-only-credential", json.dumps(nested)) + self.assertEqual(relay.reply_facts("/messages", b'{"type":"message","content":[],"error":{"code":[],"type":{},"message":17}}', False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b'{"error":{"code":502,"message":"m","detail":{"key":"x"}}}', False), + {"error": {"code": 502, "message": "m"}}) + + def test_of_several_choices_errors_the_top_level_one_else_the_first_is_kept(self): + two = [{"index": 0, "finish_reason": "error", "error": {"code": 429, "message": "rate limited"}}, + {"index": 1, "finish_reason": "error", "error": {"code": 503, "message": "unavailable"}}] + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps({"choices": two}).encode(), False)["error"], + {"code": 429, "message": "rate limited", "choice": 0}) + top = {"choices": two, "error": {"code": 500, "message": "whole"}} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(top).encode(), False)["error"], {"code": 500, "message": "whole"}) + + def test_a_merged_job_keeps_each_shards_whole_summary(self): + """A shard's summary holds its benchmark relay's calls too: a merge made of its trials alone would drop + them, and an ``unknown`` would read true.""" + pin = {"m": ["prov-a"]} + trial = {"standard": {"served": job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov A"}}], pin), "shard_job": "s1"}} + bench = job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov B"}}], pin) + shard = {"name": "s1", "config": {"standard": {"served_by": pin}}, "result": {"served": job.served_total([trial], bench)}} + self.assertEqual(merge._served([shard], [trial]), shard["result"]["served"]) + old = {"name": "s2", "config": {"standard": {"served_by": pin}}, "result": {}} # a shard written before: its trials + self.assertEqual(merge._served([old], [dict(trial, standard=dict(trial["standard"], shard_job="s2"))])["held"], {"m": True}) + + def test_a_control_is_refused_a_pin_on_its_simulated_users_model(self): + b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing"}}, + simulator={"model": "sim"}) + with self.assertRaisesRegex(run.Refused, "names sim, which this run does not call"): + run.Run(b, definitions.control(b, "nop"), model="sim", provider="openrouter", env={}, + jobs_root=self.tmp, say=lambda _: None, served_by={"sim": ["prov-a"]}) + + def test_the_relays_own_command_refuses_another_provider_and_a_model_it_does_not_send(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("review-test-key") + for base, model, extra in (("https://api.openai.com/v1", "m", []), ("https://openrouter.ai/api/v1", "unused", []), + ("http://127.0.0.1:9/v1", "m", [])): + with self.subTest(base=base, model=model): + out = io.StringIO() + with mock.patch.object(relay, "Relay") as fake, redirect_stdout(out): + code = relay.serve(["--base", base, "--model", "m", "--key-file", key, "--record", self.tmp, + "--served-by", model + "=prov-a", *extra]) + self.assertEqual(code, 2) + self.assertIn("--served-by", json.loads(out.getvalue())["error"]) + fake.assert_not_called() + + +class TheMerge(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing said"}}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def shards(self, *pins): + jobs = [run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=os.path.join(self.tmp, "jobs"), + env={}, say=lambda s: None, shard=(i, len(pins)), launch="L", name=f"run-shard-{i}of{len(pins)}") + for i in range(1, len(pins) + 1)] + for job_dir, pin in zip(jobs, pins): + path = os.path.join(job_dir, "config.json") + with open(path) as f: + cfg = json.load(f) + cfg["standard"]["served_by"] = pin + with open(path, "w") as f: + json.dump(cfg, f) + return jobs + + def test_shards_of_different_pins_are_refused(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + self.assertEqual(merge.check(jobs, force=True)["differ"][0].split(":")[0], "serving providers") + jobs = self.shards({"m-1": ["prov-a", "prov-b"]}, {"m-1": ["prov-b", "prov-a"]}) # the order is the pin's + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + + def test_equal_pins_merge_whatever_the_order_of_their_models(self): + jobs = self.shards({"a": ["prov-a"], "b": ["prov-b"]}, {"b": ["prov-b"], "a": ["prov-a"]}) + self.assertEqual(merge.check(jobs)["differ"], []) + + def test_a_forced_merge_of_different_pins_holds_no_pin(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + for job_dir, who in zip(jobs, ("Prov A", "Prov B")): + trial = sorted(d for d in os.listdir(job_dir) if "__u" in d)[0] + path = os.path.join(job_dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["served"] = {"calls": {who: 1}, "unnamed": 0, "held": {"m-1": True}} + with open(path, "w") as f: + json.dump(res, f) + out = os.path.join(self.tmp, "merged") + merge.merge(jobs, out, force=True) + with open(os.path.join(out, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + with open(os.path.join(out, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_served_by_constraints.py b/app/tests/test_standard_served_by_constraints.py new file mode 100644 index 0000000..b606c49 --- /dev/null +++ b/app/tests/test_standard_served_by_constraints.py @@ -0,0 +1,172 @@ +"""Provider pins and machine constraints on the same run and recipe. + +These intersections need both features: neither side's tests exercise them +together. The fixture driver scores locally; no endpoint or engine is used. +""" +from __future__ import annotations + +import io +import json +import os +from pathlib import Path +import shlex +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from nullius import constraints, progress +from nullius.standard import cli, container, definitions, install, job, merge, recipe, run +from tests import test_standard_recipe_served_by as recipes + + +class Together(unittest.TestCase): + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(mock.patch.object(install, "HOME", str(self.tmp / "install"))) + self.addCleanup(container.cap, {}) + self.b = recipes.judged() + self.a = definitions.get("agent", "fake", recipes.ROOT) + self.pin = {"m-judge": ["prov-a"]} + + def make(self, **kw): + values = dict(model="m", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, + judge_model="m-judge", served_by=self.pin, unsandboxed=True, + jobs_root=str(self.tmp), say=lambda _: None) + values.update(kw) + return run.Run(self.b, self.a, **values) + + def test_the_cli_records_units_constraints_and_pins_together(self): + units = self.tmp / "units.txt" + units.write_text("good\n") + self.enterContext(mock.patch.object(cli, "_bench", return_value=self.b)) + self.enterContext(mock.patch.object(cli, "_exit_on_term")) + self.enterContext(mock.patch.object(cli.settings, "load")) + self.enterContext(mock.patch.object(progress, "ECHO", None)) + self.enterContext(mock.patch.object(run.relay.Relay, "start", lambda r: r)) + self.enterContext(mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9")) + self.enterContext(mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"})) + seen = [] + proc = run.jsonl.Proc + + def driver(cmd, cwd, env, log, **kw): + seen.append(env) + return proc(cmd, cwd, env, log, **kw) + + self.enterContext(mock.patch.object(run.jsonl, "Proc", driver)) + for i, extra in enumerate(([], ["--from-run"])): + out = io.StringIO() + with self.subTest(extra=extra), redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--jobs", str(self.tmp), "--name", f"job-{i}", + "--units", str(units), "-j", "4", "--constraint", "concurrency=1", + "--constraint", "gpu_ids=", "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", *extra]) + self.assertEqual(code, 0, out.getvalue()) + path = Path(out.getvalue().strip().splitlines()[-1]) + cfg = json.loads((path / "config.json").read_text()) + self.assertEqual(cfg["standard"]["units_file"], run.units_file(str(units))) + self.assertEqual(cfg["standard"]["constraints"], {"concurrency": 1, "gpu_ids": []}) + self.assertEqual(cfg["standard"]["served_by"], self.pin) + self.assertEqual(cfg["n_concurrent_trials"], 1) + self.assertEqual(sorted(p.name for p in path.iterdir() if "__u" in p.name), ["good__u0000"]) + self.assertEqual(json.loads((path / "result.json").read_text())["served"]["held"], {"m-judge": None}) + self.assertTrue(seen) + self.assertTrue(all(env["CUDA_VISIBLE_DEVICES"] == "" for env in seen)) + + def test_constraint_and_host_refusals_precede_pin_and_endpoint_checks(self): + with mock.patch.object(run, "provider_endpoint") as endpoint: + with self.assertRaisesRegex(run.Refused, "constraints:.*at least 1"): + self.make(constraints={"concurrency": 0}, served_by={"stray": ["prov-a"]}) + self.b = dict(self.b, host={"gpu": {"check": ["nvidia-smi"]}}, + options={"compute": {"default": "gpu", "values": ["gpu"], "needs": {"gpu": {"host": ["gpu"]}}}}) + with self.assertRaisesRegex(run.Refused, "allows Nullius no GPU"): + self.make(constraints={"gpu_ids": []}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + with self.assertRaisesRegex(run.Refused, "--served-by.*does not call"): + self.make(constraints={"gpu_ids": [0]}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + endpoint.assert_not_called() + out = io.StringIO() + with mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_exit_on_term"), \ + mock.patch.object(cli.settings, "load"), mock.patch.object(run, "run_and_report") as start, redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--constraint", "concurrency=0", "--served-by", "bad slug"]) + self.assertEqual(code, 2) + self.assertIn("refused: constraints:", out.getvalue()) + start.assert_not_called() + + def test_a_valid_pin_still_requires_enforced_machine_caps(self): + r = self.make(constraints={"cpus": 1}) + with mock.patch.object(constraints, "scope_check", return_value={"why": "unavailable", "enforced": [], "refused": {}}), \ + mock.patch.object(install, "packages") as setup, self.assertRaisesRegex(run.Refused, "systemd does not enforce"): + r.ready() + setup.assert_not_called() + + def test_the_container_relay_takes_both_caps_and_pins(self): + self.enterContext(mock.patch.object(container, "_security", "")) + with mock.patch.object(container, "unusable", return_value=None): + r = self.make(backend="container", constraints={"cpus": 1, "memory_mb": 512}) + argv = container.relay_argv("img", name="relay", job="job", app="/app", record="/rec", key_file="/key", + base="https://openrouter.ai/api/v1", model="m", routes={"u0-judge": "m-judge"}, serving=r.serving) + self.assertEqual(argv[argv.index("--cpus") + 1], "1") + self.assertEqual(argv[argv.index("--memory") + 1], "512m") + self.assertEqual(argv[argv.index("--served-by") + 1], "m-judge=prov-a") + with mock.patch.object(r, "scope_ready", return_value={}), \ + mock.patch.object(container, "base_tag", return_value="img"), \ + mock.patch.object(container, "present", return_value=True), \ + mock.patch.object(container, "base_image", return_value="img"), \ + mock.patch.object(container, "unenforced", return_value={"cpus": "max 100000"}), \ + self.assertRaisesRegex(run.Refused, "engine does not apply"): + r.ready() + + def test_each_compiled_shard_has_its_constraints_and_each_called_models_pin(self): + given = {"local": {"concurrency": 1}, "127.0.0.1": {"cpus": 2, "gpu_ids": []}} + with mock.patch.dict(os.environ, {constraints.SETTING: json.dumps(given)}): + out = recipe.compile(recipes.a_recipe(where=list(given), others=["fake", "nop"], + model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), recipes.defs(self.b)) + self.assertEqual(out["errors"], []) + self.assertEqual(out["served_models"]["model"], "openai/gpt-4o-mini") + for part in out["shards"]: + for line in part["command"].splitlines(): + words = shlex.split(line) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--constraint"], + constraints.cli_words(given[part["where"]])[1::2]) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--served-by"], + ["openai/gpt-4o-mini=prov-a"]) + self.assertLess(words.index("--constraint"), words.index("--served-by")) + + def test_builder_keeps_both_validation_errors_in_main_first_order(self): + with mock.patch.dict(os.environ, {constraints.SETTING: '{"local": {"cpus": 0}}'}): + out = recipe.compile(recipes.a_recipe(model="openai/gpt-4o-mini", + served_by={"model": ["prov-a"], "judge": ["prov-b"]}), recipes.defs(self.b)) + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"], recipe.STEP["model"]]) + self.assertIn("HOST_CONSTRAINTS", out["errors"][0]["message"]) + self.assertIn("pins openai/gpt-4o-mini twice", out["errors"][1]["message"]) + self.assertEqual((out["script"], out["parts"]), ("", [])) + + def test_merge_allows_machine_limits_to_differ_but_still_checks_pins(self): + paths = [] + for i in (1, 2): + path = self.tmp / f"shard-{i}" + path.mkdir() + job.write_config(str(path), job=path.name, bench=self.b, other=self.a, model="m", provider="openrouter", + dataset="all", units=[], concurrency=i, granted=True, launch="launch", + shard={"index": i, "of": 2, "units_total": 0, "planned": []}, + constraints={"concurrency": i}, served_by=self.pin) + (path / "result.json").write_text(json.dumps({"served": {"calls": {"Prov A": 1}, "unnamed": 0, + "held": {"m-judge": True}}})) + paths.append(str(path)) + self.assertEqual(merge.check(paths)["differ"], []) + merge.merge(paths, str(self.tmp / "joined")) + cfg = json.loads((self.tmp / "joined/config.json").read_text())["standard"] + self.assertEqual((cfg["constraints"], cfg["served_by"]), ({"concurrency": 1}, self.pin)) + self.assertEqual(json.loads((self.tmp / "joined/result.json").read_text())["served"]["held"], {"m-judge": True}) + path = Path(paths[1]) / "config.json" + cfg = json.loads(path.read_text()) + cfg["standard"]["served_by"] = {"m-judge": ["prov-b"]} + path.write_text(json.dumps(cfg)) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(paths) + merge.merge(paths, str(self.tmp / "forced"), force=True) + self.assertEqual(json.loads((self.tmp / "forced/result.json").read_text())["served"]["held"], {"m-judge": "unknown"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index 7d84a04..751625c 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -15,7 +15,7 @@ from types import SimpleNamespace from nullius.runs import launches, parse -from nullius.standard import job, merge, shard +from nullius.standard import job, merge, prove, shard BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} @@ -61,14 +61,15 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local"): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", bench_options=None): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) os.makedirs(d) planned = list(units) if planned is None else planned job.write_config(d, job=name, bench=bench, other=other, model=model, provider="openai", dataset="all", units=planned, - concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend) + concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend, + bench_options=bench_options) results = [] for k, u in enumerate(units): place = int(u.split("-")[1]) @@ -80,7 +81,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": 1.0 if place % 2 else 0.0}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) + job.write_result(d, name, t0, t0 + 10 * (min(index, 9) + 1), results, len(planned), budget=budget) return d @@ -199,6 +200,36 @@ def test_units_no_shard_ran_are_said(self): j = parse.parse_job(got["out"]) self.assertEqual((j.n_total_trials, len(j.trials)), (len(self.split[1]) + len(planned), len(self.split[1]) + len(planned) - 1)) + def test_a_launch_of_one_unit_per_shard_merges_and_says_only_the_units_it_lacks(self): + n = 2 ** 64 + units = [f"t-{i}" for i in range(10)] + dirs = [write_shard(self.tmp, f"one-{u}", [u], shard.of(u, n), n) for u in units[:-1]] + got = merge.merge(dirs, os.path.join(self.tmp, "out")) + self.assertEqual((got["trials"], got["notes"]), (9, ["1 of the dataset's 10 units are not in the merge"])) + + def test_shards_run_with_different_benchmark_options_keep_each_its_own(self): + same = {"websites": "none", "gitlab": "none"} + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, bench_options=same), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example")), + write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)] + std = read(os.path.join(merge.merge(dirs, os.path.join(self.tmp, "out"))["out"], "config.json"))["standard"] + self.assertNotIn("benchmark_options", std) + self.assertEqual([m["benchmark_options"]["gitlab"] for m in std["merged_from"]], ["none", "gitlab.example", "none"]) + alike = [write_shard(self.tmp, f"a{i}", self.split[i], i, 3, bench_options=same) for i in (1, 2, 3)] + std = read(os.path.join(merge.merge(alike, os.path.join(self.tmp, "alike"))["out"], "config.json"))["standard"] + self.assertEqual(std["benchmark_options"], same) + self.assertNotIn("benchmark_options", std["merged_from"][0]) + + def test_a_merge_of_merges_keeps_the_options_of_its_inner_shards(self): + same = {"websites": "none", "gitlab": "none"} + inner = merge.merge([write_shard(self.tmp, "s1", self.split[1], 1, 3), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example"))], + os.path.join(self.tmp, "inner"))["out"] + outer = merge.merge([inner, write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)], + os.path.join(self.tmp, "outer"), force=True)["out"] + self.assertEqual(prove._options_of(read(os.path.join(outer, "config.json"))), + [dict(same, gitlab="gitlab.example"), same, {}]) + def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): dirs = self.shards() os.makedirs(os.path.join(self.tmp, "taken", "x")) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267..ea68afc 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -6,6 +6,7 @@ import os import tempfile import unittest +from unittest import mock from nullius.standard import job from nullius.traces import trace @@ -139,6 +140,31 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) + def test_a_trace_that_fails_leaves_the_result_and_says_its_error(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + with mock.patch.object(job, "trace_from_run", side_effect=KeyError("the task's own words")): + path = self.write(AGENT, traj) + agent = os.path.dirname(path) + self.assertFalse(os.path.exists(path)) + with open(os.path.join(os.path.dirname(agent), "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["rewards"], {"reward": 1.0}) + with open(os.path.join(agent, "trace.error.txt")) as f: + self.assertEqual(f.read(), "KeyError\n") + self.assertTrue(os.path.exists(os.path.join(agent, "trajectory.json"))) + + def test_a_trace_error_that_cannot_be_written_leaves_the_result(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + write = job._write + + def refusing(path, text): + if os.path.basename(path).startswith("trace"): + raise PermissionError(path) + write(path, text) + + with mock.patch.object(job, "_write", refusing): + path = self.write(AGENT, traj) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(os.path.dirname(path)), "result.json"))) + class Backfill(unittest.TestCase): """A trial written before traces gets one from the files it kept, and nothing else changes.""" diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index d7c93da..594fd76 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -115,7 +115,7 @@ def tree(job_dir): mock.patch.object(install, "upstream", side_effect=install.InstallFailed("git fetch: exit 128")), \ self.assertRaises(run.Refused): preparing.start() - self.assertEqual(tree(preparing.job_dir), [".work", "progress.jsonl"]) + self.assertEqual(tree(preparing.job_dir), [".work", ".work/job.lck", "progress.jsonl"]) self.assertEqual(tree(unknown.job_dir), tree(preparing.job_dir)) self.assertIn("nope", progress.refusal(progress.read(unknown.job_dir))) @@ -347,6 +347,22 @@ def test_the_listed_units_run_under_the_machine_s_constraints(self): cfg = read(os.path.join(job, "config.json")) self.assertEqual((cfg["standard"]["units_file"]["units"], cfg["standard"]["constraints"], cfg["n_concurrent_trials"]), (["slow", "good"], {"concurrency": 1}, 1)) + def test_the_command_records_both_the_units_file_and_the_serving_pin(self): + path = self.file("good\n") + self.b["score"] = dict(self.b["score"], judge={"model": "m-judge", "says": "a judge"}) + # The fixture scores locally; no relay listener is needed. + for extra in ([], ["--from-run"]): + with self.subTest(extra=extra), mock.patch.object(run.relay.Relay, "start", lambda r: r), \ + mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9"), \ + mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"}): + code, out = self.cli("--units", path, "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", "--name", "pinned" + "".join(extra), *extra) + self.assertEqual(code, 0, out) + job = out.strip().splitlines()[-1] + standard = read(os.path.join(job, "config.json"))["standard"] + self.assertEqual(standard["units_file"], run.units_file(path)) + self.assertEqual(standard["served_by"], {"m-judge": ["prov-a"]}) + self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000"]) def test_a_list_the_run_cannot_take_is_refused_with_exit_2(self): for text, said in (("good\ngood\n", "more than once"), ("# none\n", "lists no unit"), ("good\nnope\n", "nope")): diff --git a/app/tests/test_standard_versions.py b/app/tests/test_standard_versions.py index 898a0dc..1c86087 100644 --- a/app/tests/test_standard_versions.py +++ b/app/tests/test_standard_versions.py @@ -217,7 +217,7 @@ def test_new_version_copies_the_default_and_refuses_a_duplicate(self): ''' TOY = dict(BENCH, upstream={"license": "MIT"}, default="1", controls={"nop": {"source": "nullius", "from": "nothing runs"}}, - versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 2}], + versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 3}], "driver": {"command": ["python3", "{here}/driver2.py"]}, "proven": {}}, "1": {"proven": {"all": {"units": {"units": 2, "passed": "2 ids", "on": "2026-10-12"}}}}}) diff --git a/app/ui_tests/test_served_by_builder.py b/app/ui_tests/test_served_by_builder.py new file mode 100644 index 0000000..11e82fa --- /dev/null +++ b/app/ui_tests/test_served_by_builder.py @@ -0,0 +1,140 @@ +"""The Model step's Served by: the route that lists a model's providers on +OpenRouter (local, form-token guarded, asked as the model list is), and the +recipe's ``served_by`` in the builder's core script (its fields on +OpenRouter alone, kept by a saved recipe, dropped with another provider).""" +import json +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from .builder_support import CompilerCase +from .test_routes import web + +LIST = [{'slug': 'deepinfra/fp8', 'name': 'DeepInfra', 'quantization': 'fp8'}, {'slug': 'together', 'name': 'Together', 'quantization': ''}] + + +class ProviderList(unittest.TestCase): + def setUp(self): + self.client = web.app.test_client() + with self.client.session_transaction() as session: + session['csrf'] = 'served-test-token' + self.headers = {'X-CSRF-Token': 'served-test-token'} + + def ask(self, body, **kw): + return self.client.post('/new-run/endpoints', json=body, headers=kw.pop('headers', self.headers), **kw) + + def test_a_models_providers_on_openrouter(self): + with patch('web.builder.providers.list_endpoints', return_value=LIST) as fetch: + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {'ok': True, 'endpoints': LIST}) + fetch.assert_called_once_with('openrouter', 'meta-llama/llama-3.3-70b-instruct') + + def test_a_list_that_cannot_be_had_is_no_empty_success(self): + with patch('web.builder.providers.list_endpoints', side_effect=ValueError("Provider list unavailable. Enter a provider's slug by hand.")): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 502) + self.assertEqual(response.json, {'ok': False, 'endpoints': [], 'message': "Provider list unavailable. Enter a provider's slug by hand."}) + + def test_no_key_asks_without_one(self): + """OpenRouter's endpoint list is public: with no key set, it is asked with no Authorization header.""" + sent = [] + + def get(url, headers, opener=None): + sent.append((url, headers)) + return json.dumps({'data': {'endpoints': [{'provider_name': 'Together', 'tag': 'together'}]}}).encode() + env = {k: v for k, v in os.environ.items() if k != 'OPENROUTER_API_KEY'} + with patch.dict(os.environ, env, clear=True), patch('web.builder.providers._get', side_effect=get): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.json['endpoints'], [{'slug': 'together', 'name': 'Together', 'quantization': ''}]) + self.assertEqual(sent, [('https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints', {})]) + + def test_guards_and_another_provider_do_not_fetch(self): + with patch('web.builder.providers.list_endpoints') as fetch: + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, headers={}).status_code, 403) + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, environ_overrides={'REMOTE_ADDR': '192.0.2.4'}).status_code, 403) + for body in ({'provider': 'openai', 'model': 'gpt-4o'}, {'provider': 'openrouter'}, {'provider': 'openrouter', 'model': ' '}, []): + with self.subTest(body=body): + self.assertEqual(self.ask(body).status_code, 400) + fetch.assert_not_called() + + +class TheStep(unittest.TestCase): + """What the Model step's script draws and what the compile route hands it.""" + SCRIPT = (Path(__file__).resolve().parents[1] / 'static' / 'builder' / 'command-builder.js').read_text() + + def test_served_by_holds_no_sentence_and_its_help_is_a_tooltip(self): + field = self.SCRIPT[self.SCRIPT.index('const servedField'):self.SCRIPT.index('const servedCache')] + self.assertNotIn('', field) + self.assertIn('title="', field) + + def test_the_compile_route_gives_each_fields_model_as_the_run_sends_it(self): + client = web.app.test_client() + with client.session_transaction() as session: + session['csrf'] = 'served-test-token' + recipe = {'version': 1, 'harness': 'standard', 'name': 'x-0101', 'title': 'x', 'benchmark': 'longmemeval', + 'datasets': ['longmemeval_s'], 'others': ['original'], 'model_choice': 'paper', 'provider': 'openrouter', + 'model': 'gpt-4o', 'limit': 1, 'concurrency': 1, 'where': 'local'} + response = client.post('/new-run/runner', json={'recipe': recipe}, headers={'X-CSRF-Token': 'served-test-token'}) + self.assertEqual(response.json['served_models']['model'], 'openai/gpt-4o-2024-08-06') + + def test_the_keyboard_stays_on_the_model_field_once_the_form_is_drawn_again(self): + self.assertIn("requestAnimationFrame(()=>{const f=root.querySelector(`[data-field=\"${d.field}\"]`);if(f&&document.activeElement!==f)f.focus();})", + self.SCRIPT) + + +class ServedInTheCore(CompilerCase): + """command-builder-core.js: which model fields take a Served by, and the recipe's ``served_by``.""" + + def test_the_fields_show_on_openrouter_alone(self): + out = self.js('const b=C.standard().benchmarks.find(x=>x.simulator&&x.judge)||C.standard().benchmarks.find(x=>x.judge);' + 'const r=C.createStandard(b.key);r.model_choice="any";const got={};' + 'for(const p of ["openrouter","openai","custom"]){C.setChoice(r,p);got[p]=C.servedRoles(r);}' + 'process.stdout.write(JSON.stringify([got,!!b.simulator]));') + roles, simulated = out + self.assertEqual(roles['openrouter'], ['model'] + (['simulator'] if simulated else []) + ['judge']) + self.assertEqual((roles['openai'], roles['custom']), ([], [])) + + def test_a_saved_recipe_keeps_its_pins_and_another_provider_drops_them(self): + raw = {'version': 1, 'harness': 'standard', 'benchmark': '', 'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct', + 'served_by': {'model': ['deepinfra/fp8', ' together '], 'judge': ['bad slug'], 'agent': ['x'], 'simulator': []}} + out = self.js('const b=C.standard().benchmarks[0];const raw=' + json.dumps(raw) + ';raw.benchmark=b.key;' + 'const r=C.fromJSON(JSON.stringify(raw));const kept=JSON.parse(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model","a, b");const typed=r.served_by.model;C.setServed(r,"model","");const emptied=!(r.served_by||{}).model;' + 'C.setServed(r,"model",["a"]);C.setChoice(r,"openai");' + 'process.stdout.write(JSON.stringify([kept,typed,emptied,r.served_by===undefined]));') + # kept as written, the judge's bad slug included: the server refuses it in sight (test_a_pin_is_kept_as_written...) + self.assertEqual(out, [{'model': ['deepinfra/fp8', 'together'], 'judge': ['bad slug']}, ['a', 'b'], True, True]) + + def test_a_pin_goes_with_the_model_it_was_set_for(self): + out = self.js('const b=C.standard().benchmarks.find(x=>x.judge)||C.standard().benchmarks[0];const r=C.createStandard(b.key);C.setChoice(r,"openrouter");' + 'const seen=[];C.setServed(r,"model",["a"]);C.setServed(r,"judge",["j"]);' + 'C.setModelOf(r,"model",r.model);seen.push(JSON.stringify(r.served_by));' + 'C.setModelOf(r,"model","meta-llama/llama-3.3-70b-instruct");seen.push(JSON.stringify(r.served_by));' + 'C.setModelOf(r,"judge_model","openai/gpt-4o");seen.push(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model",["a"]);C.setModelChoice(r,r.model_choice);seen.push(JSON.stringify(r.served_by));' + 'C.setModelChoice(r,C.paperOffered(r)?"paper":"leaderboard");seen.push(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model",["a"]);const other=C.standard().benchmarks.find(x=>x.key!==b.key);C.setStandardBenchmark(r,other.key);seen.push(JSON.stringify(r.served_by||null));' + 'process.stdout.write(JSON.stringify(seen));') + self.assertEqual(out, ['{"model":["a"],"judge":["j"]}', '{"judge":["j"]}', 'null', '{"model":["a"]}', 'null', 'null']) + + def test_a_pin_is_kept_as_written_for_the_server_to_refuse(self): + raw = {'version': 1, 'harness': 'standard', 'benchmark': '', 'provider': 'openrouter', 'served_by': {'model': ['deepinfra', 'bad slug']}} + out = self.js('const raw=' + json.dumps(raw) + ';raw.benchmark=C.standard().benchmarks[0].key;const r=C.fromJSON(JSON.stringify(raw));' + 'process.stdout.write(JSON.stringify([r.served_by,C.servedList("a,,b"),C.servedList(" , "),C.servedList("")]));') + self.assertEqual(out, [{'model': ['deepinfra', 'bad slug']}, ['a', '', 'b'], [], []]) + + def test_the_list_is_of_the_model_the_run_sends(self): + out = self.js('process.stdout.write(JSON.stringify([C.servedModelOf("model","gpt-4o",{served_models:{model:"openai/gpt-4o-2024-08-06"}}),' + 'C.servedModelOf("judge","gpt-4o",null),C.servedModelOf("judge","gpt-4o",{served_models:{judge:null}})]));') + self.assertEqual(out, ['openai/gpt-4o-2024-08-06', 'gpt-4o', 'gpt-4o']) + + def test_a_recipe_with_no_pin_is_saved_as_before(self): + out = self.js('const b=C.standard().benchmarks[0];const r=C.createStandard(b.key);C.setChoice(r,"openrouter");' + 'const again=C.fromJSON(JSON.stringify(r));process.stdout.write(JSON.stringify(["served_by" in r,"served_by" in again]));') + self.assertEqual(out, [False, False]) + + +if __name__ == '__main__': + unittest.main() diff --git a/app/web/builder.py b/app/web/builder.py index cacd9a6..d09d080 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -93,7 +93,8 @@ def new_run_runner(): abort(413) if recipe.get("harness") == standard_recipe.HARNESS: out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, unpriced=execute.unpriced) - return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm")} + # served_models: each model field's model as the run sends it, which Served by lists the providers of + return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm", "served_models")} return {"errors": [{"step": providers.STEP["runner"], "message": providers.GONE}], "notes": [], "review": []}, 404 @@ -396,6 +397,24 @@ def new_run_registry(): return _REGISTRY_SCAN +@app.route("/new-run/endpoints", methods=["POST"]) +def new_run_endpoints(): + """The serving providers OpenRouter lists for one model, for the Model step's Served by; asked as the + model list is, with the key kept on this machine.""" + shared._require_csrf() + if not shared_local_client(): + abort(403) + body = request.get_json(silent=True) + provider = body.get("provider") if isinstance(body, dict) else None + model = body.get("model") if isinstance(body, dict) else None + if providers.split(provider)[0] != "openrouter" or not isinstance(model, str) or not model.strip(): + return jsonify(ok=False, endpoints=[], message="Only OpenRouter routes a model to several providers: choose it and a model."), 400 + try: + return jsonify(ok=True, endpoints=providers.list_endpoints(provider, model)) + except ValueError as error: + return jsonify(ok=False, endpoints=[], message=str(error)), 502 + + @app.route("/new-run/models", methods=["POST"]) def new_run_models(): """Return the selected provider's catalogue, keeping its key on this machine.""" diff --git a/docs/builder.md b/docs/builder.md index 429985a..de7e986 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -26,7 +26,7 @@ Then [read its results](results.md). | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | -| Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled | +| Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, under each model field when the provider is OpenRouter: the serving providers that may answer that model, typed as slugs or added from the model's providers, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit; a [list of units](cli/run.md#a-list-of-units) is set only on the command line | | Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | diff --git a/docs/cli/read-jobs.md b/docs/cli/read-jobs.md index 86ca991..c690a67 100644 --- a/docs/cli/read-jobs.md +++ b/docs/cli/read-jobs.md @@ -56,7 +56,7 @@ nullius standard traces [--dry-run] ## Give past standard runs a trace -A standard run writes `agent/trace.json` as it ends. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. +A standard run writes `agent/trace.json` as it ends; one it cannot build leaves `agent/trace.error.txt`, the error's type, and the trial's result as it is. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. ```bash title="See what it would write, then write it" nullius standard traces ./jobs --dry-run diff --git a/docs/cli/run.md b/docs/cli/run.md index 8414a44..95e2112 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -6,6 +6,7 @@ One benchmark and one agent, capability, control or setup, into a job. nullius standard run [@version] [--model M] [--dataset D] [--units FILE] [--limit N] [-j N] \ [--unsandboxed | --backend container] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE]... \ + [--served-by [MODEL=]SLUG[,SLUG...]]... \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -32,6 +33,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | | `--max-usd USD`, `--max-usd-per-unit USD` | A spending cap for the run, for each unit: [below](#a-spending-cap) | | `--constraint KEY=VALUE` | What the machine allows the run (repeatable): `cpus`, `memory_mb`, `processes`, `cpu_set`, `concurrency`, `gpu_ids` (indices, empty for none); [constraints](../settings/run-hosts.md#constraints). Containers take them as flags, the run's processes on the machine as a systemd scope. A recipe passes each machine's own | +| `--served-by [MODEL=]SLUG[,SLUG...]` | On OpenRouter, the serving providers that may answer a model (repeatable): [below](#serving-providers) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | | `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run | | `--judge-model` | Instead of the official judge | @@ -88,6 +90,27 @@ HumanEval/12 | Where it is read | On the machine the command runs on, from the directory it starts in, with `--from-run` or without | | The builder | A recipe has no field for a list | +## Serving providers + +On OpenRouter one model is served by several providers. They may differ in precision, speed and price. `--served-by deepinfra` lets only DeepInfra answer the run's model; `--served-by m=deepinfra/fp8,together` lets DeepInfra's fp8 endpoint answer the model `m`, then Together when it cannot. + +```bash title="One provider for the run's model" +nullius standard run humaneval reference --provider openrouter --model meta-llama/llama-3.3-70b-instruct --served-by deepinfra --unsandboxed +``` + +| Rule | Detail | +| --- | --- | +| A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | +| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings. A model the benchmark pins for the run ([standard](../standard.md), Pinned routes) keeps the benchmark's providers | +| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`). A Responses reply names the provider that served it there alone. A Messages request is sent without the header, since its reply names its provider. A model with no pin is sent as before | +| Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | +| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does, but for a model the benchmark pins ([standard](../standard.md), A refused route) | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record, but for a failed Responses object's cost, and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a model the benchmark pins and a 200 with no reply ([standard](../standard.md)) | +| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | +| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name. Where that name is not the slug once lowercased, `held` reads `unknown` though the provider served the call (`Google` for `google-vertex`, `SambaNova` for `sambanova-turbo`) | +| Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | +| The builder | *Model* step, **Served by** under each model field, on OpenRouter alone: the recipe's `served_by`, by field (`model`, `simulator`, `judge`), compiled into `--served-by` for each job that calls that model. Its list of a model's providers comes from OpenRouter (`/models/{maker}/{model}/endpoints`), asked as the model list is; when it cannot be had, a slug typed by hand is taken | +
- [The standard](../standard.md#a-run) diff --git a/docs/field-guide.md b/docs/field-guide.md index 8fbffd3..f157345 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -169,6 +169,19 @@ How the agent worked, not whether it won. Computed over layer 3. | `exception_type` | What ended the trial, if anything. It decides the infra / agent split. | measured | any | | `n_rate_limited_retries` | 429s counted in `agent/gateway.log`: on Harbor, the ones the agent's SDK retried and survived; on a standard run, every model call the relay saw refused with 429. No log reads *not recorded*, never 0. | inferred | any | +## Standard run records + +These fields are stored in the standard's job and trial files ([Run a benchmark](cli/run.md)). + +| Field | What it is | Source | Works on | +| --- | --- | --- | --- | +| `standard.units_file` | In `config.json`: the units file's name, sha256 and ordered ids, before sharding and the limit. Units run in dataset order; merging requires the same listed units, even with `--force`, and keeps the first shard's file record. | measured | standard | +| `standard.constraints` | In the same config: what the machine allowed the run, when set. These values can coexist with a units file and serving pins. Shards may have different machine constraints; merging keeps the first shard's config values. | measured | standard | +| `standard.served_by` | In the same config: the allowed OpenRouter provider slugs by model, in the order tried. A job can record both its units file and its serving pins; merging different pins requires `--force`. | measured | standard | +| `served / standard.served` | In the job's `result.json` and each trial's `result.json`, respectively: answered calls by provider (`calls`), calls naming none (`unnamed`), and each pinned model's `held`. Counts cover every relay route; the job also includes the benchmark's own relay. Tokens and cost retain their existing scope. | measured | standard | +| `held` | `true` when every answered call names an allowed provider, `unknown` for an absent or different name, and `null` for no answered calls. Names are lowercased with spaces as dashes and compared with the slug before `/`; endpoint variants cannot be verified. Merging different pins marks the affected models `unknown`. | inferred | standard | +| `request.routing / request.routing_asked / reply.served_by / reply.end / reply.error` | In `agent/calls.jsonl`: routing sent, the caller's routing when replaced, the answering provider, the reply's ending and any error inside a 200 response. These credential-cleaned fields leave the call's status and the unit's verdict unchanged. | measured | standard | + ## Trajectory metrics, inside a single run {#trajectory} | Field | What it is | Source | Works on | diff --git a/docs/settings/model-providers.md b/docs/settings/model-providers.md index efcd70e..b986737 100644 --- a/docs/settings/model-providers.md +++ b/docs/settings/model-providers.md @@ -6,7 +6,7 @@ The key or endpoint a run's model calls go to. | --- | --- | --- | | TypeSafe AI | `TYPESAFE_API_KEY` | System One judgement only | | OpenAI | `OPENAI_API_KEY` | api.openai.com | -| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed | +| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed; a run may choose the serving providers of a model: **Served by** on the builder's Model step, or [`--served-by`](../cli/run.md#serving-providers) | | Anthropic | `ANTHROPIC_API_KEY` | Standard runs only, through the relay | | OpenAI-compatible | A name, a URL, an optional key, per endpoint | Ollama, vLLM, LM Studio or a hosted service, API path included | diff --git a/docs/standard.md b/docs/standard.md index f6f0068..8097f5b 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -141,14 +141,18 @@ record); verifier/reward.txt and score.json | Rule | Detail | | --- | --- | | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | -| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | +| A relay per unit | Its record is the trial's `.work/relay` (an earlier run's under the same job name is moved aside to `.work/earlier.`, with its `result.json`, `verifier/` and `agent/`; a run holds its job name until its process ends, and one under a name another still holds is refused before it writes anything); the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A model call answered 200 with no reply in it, not streamed, reaches its caller and the record as a 502, or the 429 or 402 of an error it held, that says only its kind; the record keeps `answered: 200`. A call refused with 400, 413 or 422 keeps, as `images`, each inline image it sent by its declared type, the type its bytes show whole and its size, never a byte | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | -| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a unit a guard's stop ended keeps the score its driver gave, its own last call's 429 notwithstanding; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | +| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Unless the benchmark's `limits.run_again` is `false`, each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a route the definition pins (A refused route) and a 200 with no reply (A relay per unit) | +| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | +| Pinned routes | On OpenRouter, a model the definition pins (`setups.original.pin` in a run of the Original, `score.judge.pin` while the official judge grades the run) is pinned as `--served-by` pins it, whatever `--served-by` says of that model. The pin follows the model sent, so a model the agent shares with the judge goes to the judge's providers | +| A refused route | A call the definition pins (Pinned routes) answered 404 or 503 is refused. When a model's last call on a route was refused (each route read on its own), or any such call's `reply.served_by` shares nothing with the pin's slugs (a name that is not its slug, `Google` for `google-vertex`, fails no call on a guess), the unit fails with `ApiProviderRouteError` unless its agent's own last call failed it first, an API error outside the agent's rate, even when its code made nothing of it. A gated benchmark keeps it to the status, the pin and provider names. Three units in a row whose route has no endpoint (404) stop the run, as a refused key does | ## Safety @@ -341,8 +345,10 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | Shards by the unit's id | `sha256(unit id) mod N` (`standard/shard.py`); changing N moves most units. Order: the driver's `units`, the units `--units FILE` lists, the shard, then `--limit` (`--shard 1/2 --limit 6` is the first six units of shard 1) | | A unit keeps its place | `u0042` is the 43rd unit of the whole dataset in every shard | | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model). With `--units`, `units_total` counts the listed units. The shards' lists hold the same units; a list's file name, comments and sha256 may differ from shard to shard | -| `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, backend or N are refused unless `--force`. Shards whose lists hold different units, and two trials of one unit, are refused always. `merge` compares the units each list holds, in any order; the sha256 is kept in the record only | +| `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, serving providers, backend or N are refused unless `--force`. Shards whose lists hold different units, and two trials of one unit, are refused always. `merge` compares the units each list holds, in any order; the sha256 is kept in the record only | | `cli prove --shard I/N` | The controls and the leak check on that shard's units; each record says `shard` | +| Different options per shard | Shards of one launch may differ in `--bench-option` (a unit that needs a service and a write share of its own); the merged config keeps each shard's in `merged_from` | +| `cli prove --job nop=MERGED` | Holds a merged launch of a control to its bound instead of running it. The job and every trial's own record must name that benchmark, version and control and this job; each unit of the dataset exactly once, with the field its score kind names (a number from 0 to 1) or an error; and the units check must pass. `--dataset` named, no `--limit` or `--shard`. The record names the job and the options it ran with. The runs pages group a launch's shards by their shared `standard.launch` (`--launch`); a name like `______prove-nop` is only a convention for reading | ```bash title="Two machines, one merge" nullius standard run humaneval oracle --shard 1/2 --limit 6 # machine 1 @@ -410,15 +416,20 @@ use the source folders and repository checks. | `turn` | The transcript so far; `simulator: {model, base_url, key}` when there is one; `judge` for an upstream that grades as the conversation ends; `attempt` with `limits.attempts` | The next message, `{done: true}`, or `restart` | | `produce` | A capability benchmark's recall, `base_url` and `key` | The reading, through the relay | | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | -| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | +| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by`; `stopped`, the reason of a stop asked while the unit ran, kept as `verifier_result.stopped` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | | Field in `standard.json` | Meaning | | --- | --- | -| `score.judge` | `{model, says}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another | +| `gated` | `true` for a benchmark whose tasks are gated: a unit's failing model endpoint (the agent's, the judge's, the simulated user's) is kept as its status and class, never the provider's words, which can quote the request | +| `score.judge` | `{model, says, pin?}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another. `pin` (`{"openrouter": [provider slug]}`) pins every call sent as the official judge's model while the official judge grades the run, an agent's own included; another judge is pinned only when it is a model the setup pins | | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | +| `limits.turn_seconds` | How long the agent's turn may take (default 900); a unit's `agent_seconds` or a setup's own `turn_seconds` replaces it | | `limits.turns` | At most this many agent turns in a conversation (default 20) | +| `limits.score_seconds` | How long `score` may take (default 600). A driver that runs the unit's environment in `score`, as its null control does, sets it | +| `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | +| `limits.run_again` | `false`: a unit that ended rate-limited is not run once more after the others; its outcome stands | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -582,9 +593,11 @@ already accepts arbitrary models needs only the `original` method. | Rule | Detail | | --- | --- | | `original` driver method | A model of `models` only, by the name the upstream knows; the relay sends it as `served_as` says, with the prefix `providers` gives. A provider not in `providers` is refused. A model the upstream asks besides the answerer is a `side_models` entry. Without `models`, any model: its own agent runs as the run's model | -| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError` | +| `pin` | `{"openrouter": {model: [provider slug]}}`, by the names in `models` and `side_models`: on OpenRouter each goes to those providers only, with no fallback, so the original runs on the paper's endpoint. Other setups and agents are not pinned, but for a call sent as the official judge's model while it grades the run (`score.judge.pin`) | +| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError`, when its last reply (by when it was asked) has neither text nor a tool call; a last Responses reply answered 200 and `failed` is an API error. A runner that fails once a guard's `stop` beside the relay's records asks the unit to stop is `CancelledError` | | `units` | A setup may count its own units; `units` is sent `setup` | | `answer` | Runs the upstream's answerer through the unit's relay; `score` is sent `setup` too. Local processes or the Original setup driver container | +| `comparable` | `false` when no model of `models` is the paper's, so its runs compare with each other only; or the list of those models whose runs compare with the paper's (`paper`), when the setup runs several agents and only some are the paper's: the builder offers those alone as Like the paper, the others as Any of your choice | | Recorded | `standard.setup` in the config and each trial: its name, the model `asked` and `sent`, `comparable_with_paper` | | Attempts | `cli run --repeat K` runs each unit K times; `result.json` carries `pass_hat_k` (C(c, k) / C(n, k) averaged over the units, an errored unit left out) | @@ -627,7 +640,7 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c `cli run KEY nop` and `cli run KEY oracle` need no `--model` and no `--unsandboxed`: no model is called but a judged benchmark's judge. The config and every trial record `standard.control`. -`cli prove KEY [--dataset D] [--limit N]` runs, for each dataset: +`cli prove KEY [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop=JOB_DIR]` runs, for each dataset: | Check | Passes when | | --- | --- | @@ -636,6 +649,8 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c | `units` | Two `units` calls give the same ids: unique, plain enough to name a trial, as many as the dataset says | | `leak` | After `setup`, the oracle's answer and the lines it writes are not already in the prompt or the workspace. A reference under 12 characters is not looked for; the record says how many were skipped | +`--bench-option NAME=VALUE` sets the benchmark's own options for the units check's driver and each control's run, resolved and refused as `cli run` does them, the programs a value needs on this machine included, before anything is installed or started; the control job's config records the values. + What passed is written into `standard.json` per dataset, dated; a failing check is taken back, and a run on fewer units does not replace one on more. ```json title="standard.json · proven"