From c570c02dbb236424e1ffc9730fa14f75992aa291 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 14:54:54 +0100 Subject: [PATCH 01/24] feat(standard): pin the serving providers that may answer a model On OpenRouter one model is served by several providers, and a run could neither say which may answer nor see which one did. `--served-by [MODEL=]SLUG[,SLUG...]`, repeatable, names them for a model (the run's when MODEL= is left out). Every relay of the run, a container's sidecar included, puts `provider: {order: [...], allow_fallbacks: false}` on each request it sends as that model, on any route and in the three API shapes, in place of the caller's own routing field. A pinned Chat Completions request asks for OpenRouter's routing facts, where its reply names the provider that served it. Without the option a request goes on byte for byte as before. The option is refused before any unit when the run's provider routes no model to several (only openrouter does), when it names a model the run does not call, or when a slug is malformed or given twice. Each call's record keeps the routing sent (and the caller's own when it was replaced), the serving provider the reply names, how the reply ended and an error the reply holds under a 200, which now makes the call a failed one. The job's config records the pin by model; each trial and the job say the answered calls by serving provider and, for a pinned model, whether every one was served by a pinned provider. merge refuses shards whose pins differ, unless --force. --- app/nullius/standard/cli.py | 9 +- app/nullius/standard/container.py | 14 +- app/nullius/standard/job.py | 80 +++++- app/nullius/standard/merge.py | 11 +- app/nullius/standard/relay.py | 152 ++++++++-- app/nullius/standard/run.py | 59 +++- app/tests/test_standard_served_by.py | 402 +++++++++++++++++++++++++++ 7 files changed, 682 insertions(+), 45 deletions(-) create mode 100644 app/tests/test_standard_served_by.py diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 44759fb..e5399bc 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,7 +7,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--max-usd USD] [--max-usd-per-unit USD] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -41,6 +41,9 @@ puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option`` sets one of the agent's or capability's options (``options`` in its definition); the rest keep the vendor's defaults, and the job's config records them all. +``--served-by [MODEL=]SLUG[,SLUG...]`` lets only those serving providers answer +a model on OpenRouter, in that order (``run.served_by``); each call's record +names the provider that served it. A run says each stage before its first unit, and each unit's end, as one ``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl`` (``nullius/progress.py``): what the run's page shows while it goes. @@ -498,6 +501,7 @@ def cmd_run(a): simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, + **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) @@ -659,6 +663,9 @@ def main(argv=None): help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults") p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]", help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)") + p.add_argument("--served-by", action="append", metavar="[MODEL=]SLUG[,SLUG...]", + help="on OpenRouter, the serving providers that may answer a model, tried in this order and no " + "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd", type=float, metavar="USD", help="the run's spending cap: no call goes out once the spend plus the calls in flight would pass it") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index 1b1d8f3..0f51295 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -686,7 +686,7 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -694,7 +694,9 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi an empty folder). ``hosts``: names it resolves (``--add-host``), the machine's own for an endpoint on its loopback. ``ledger``: the file of the run's spending cap, its folder mounted so - this relay books its calls where every other relay of the job does.""" + this relay books its calls where every other relay of the job does. + ``serving``: by model, the serving providers that may answer it, as the + unit's own relay is given them (``relay.Relay``).""" a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init", *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")], @@ -714,6 +716,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] + for m, slugs in sorted((serving or {}).items()): + a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] return a @@ -976,9 +980,11 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True): + ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, + serving: Optional[Dict[str, List[str]]] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one + self.serving = dict(serving or {}) # by model, the serving providers that may answer it self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1004,7 +1010,7 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger) + hosts=self.hosts, ledger=self.ledger, serving=self.serving) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index ea8c8cd..21d0211 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -400,11 +400,68 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: return "full" if full else "partial" +def _named(name: str) -> str: + """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``Google AI Studio`` + reads ``google-ai-studio``). A run fetches no list of providers, so this is all a name is matched by.""" + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> Optional[dict]: + """Who served a unit's answered calls (each record's ``reply.served_by``): the calls by serving provider and + those that named none; and, for each model the run pinned (``serving``), whether the pin held - true when + every answered call sent as that model names a provider that reads as one of its pinned slugs (``_named``, + a slug's first part), ``unknown`` when one names none or another name, None when it has no answered call. + None for a unit whose calls name no provider in a run with no pin.""" + answered = [c for c in calls if c.get("status") == 200 and not (c.get("reply") or {}).get("error")] + by: Dict[str, int] = {} + unnamed = 0 + for c in answered: + name = (c.get("reply") or {}).get("served_by") + if isinstance(name, str) and name: + by[name] = by.get(name, 0) + 1 + else: + unnamed += 1 + if not by and not serving: + return None + out = {"calls": dict(sorted(by.items())), "unnamed": unnamed} + if serving: + held = {} + for m, slugs in sorted(serving.items()): + mine = [c for c in answered if c.get("model") == m] + pinned = {str(s).split("/")[0].lower() for s in slugs} + held[m] = None if not mine else True if all( + _named(str((c.get("reply") or {}).get("served_by") or "")) in pinned for c in mine) else "unknown" + out["held"] = held + return out + + +def served_total(results: List[dict]) -> Optional[dict]: + """The job's ``served``: its trials' added up, a pin held where it held in every trial that called its model.""" + parts = [p for p in ((r.get("standard") or {}).get("served") for r in results) if p] + if not parts: + return None + calls: Dict[str, int] = {} + for p in parts: + for name, n in (p.get("calls") or {}).items(): + calls[name] = calls.get(name, 0) + n + out = {"calls": dict(sorted(calls.items())), "unnamed": sum(p.get("unnamed") or 0 for p in parts)} + models = sorted({m for p in parts for m in (p.get("held") or {})}) + if models: + out["held"] = {} + for m in models: + said = [p["held"][m] for p in parts if (p.get("held") or {}).get(m) is not None] + out["held"][m] = None if not said else True if all(v is True for v in said) else "unknown" + return out + + def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict: - """Write one unit's trial and return its result. ``environment``: the + image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), + serving: Optional[Dict[str, List[str]]] = None) -> dict: + """Write one unit's trial and return its result. ``serving``: the run's + pin of serving providers by model; ``standard.served`` says who served + the unit's calls and whether the pin held (:func:`served`). ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began @@ -426,7 +483,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]}, "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), - **({"environment": environment} if environment else {}), **({"image": image} if image else {})}} + **({"environment": environment} if environment else {}), **({"image": image} if image else {}), + **({"served": who} if (who := served(calls, serving)) else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score @@ -464,7 +522,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, - simulator: str = "", prices: Optional[dict] = None): + simulator: str = "", prices: Optional[dict] = None, served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -473,7 +531,9 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str many times each unit was run, said only when more than once. ``judge`` and ``simulator``: the models that graded and that played the user, when the benchmark has them. ``prices``: the rows its models' costs are - estimated from and where the user's list came from, when it has any.""" + estimated from and where the user's list came from, when it has any. + ``served_by``: by model, the serving providers the run let answer it + (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -482,7 +542,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"image": image} if image else {}), **({"judge": judge} if judge else {}), **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), - **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {})}, + **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), + **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], @@ -541,11 +602,14 @@ def pass_hat(results: List[dict]) -> Optional[dict]: def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, what was spent, whether it stopped the run, the units it cut off and - those never begun.""" + those never begun. ``served``: who served its trials' calls, when a + trial says it (:func:`served`).""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) + who = served_total(results) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, - **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1) + **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {}), + **({"served": who} if who else {})}, f, indent=1) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index f539fb8..9f23261 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -9,8 +9,9 @@ the shards' with ``shard`` taken out and ``merged_from`` put in. What is merged must be one launch: the same launch id, benchmark and -version, dataset, agent or capability and model, and the same number of shards -- else it is refused, naming what differs, unless ``force``. Two trials of +version, dataset, agent or capability and model, the same serving providers +pinned for each model (``served_by``) and the same number of shards - else it +is refused, naming what differs, unless ``force``. Two trials of one unit are always refused, with the units named: one of them would be counted twice. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: the shards absent, and the units a @@ -36,7 +37,8 @@ SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")), ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")), - ("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of"))) + ("backend", ("standard", "backend")), ("serving providers", ("standard", "served_by")), + ("number of shards", ("standard", "shard", "of"))) class Refused(Exception): @@ -220,7 +222,8 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), "scored": len(scores), "mean_score": job.mean_score(results)}, - **({"budget": budget} if budget else {})}, f, indent=1) + **({"budget": budget} if budget else {}), + **({"served": who} if (who := job.served_total(results)) else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a8c5945..9c4f139 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -8,8 +8,11 @@ A reply is passed through as it arrives - a streamed one (server-sent events) byte for byte - and recorded once it is complete, one line per call in ``/.jsonl``: when it started and ended, the model, the status, -the tokens, and an abridged request and reply. Three API shapes are read, -Chat Completions, Responses and Anthropic's Messages, streamed or not. The +the tokens, and an abridged request and reply: the reply's serving provider +when it names one (``served_by``), how it ended (``end``) and an error it +holds inside a success (``error``), which makes the call a failed one. Three +API shapes are read, Chat Completions, Responses and Anthropic's Messages, +streamed or not. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the user's price list, ``nullius.prices``), marked as an estimate. @@ -24,6 +27,12 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +A run may pin, for a model, the serving providers that may answer it +(``serving``, OpenRouter's routing): the relay then puts +``provider: {order, allow_fallbacks: false}`` on every request it sends as +that model, in place of the caller's own, and records both. Without a pin a +request's body goes on as it did. + One answer is the relay's own: a run with a spending cap (``ledger``) books each call before it goes out, and a call that does not fit is not sent. Its caller gets a 402 that names the relay and the cap, and the record says @@ -138,28 +147,85 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def routing(slugs) -> dict: + """OpenRouter's routing field for a strict pin: these serving providers, + tried in this order, and no other.""" + return {"order": list(slugs), "allow_fallbacks": False} + + +def _error(e) -> Optional[dict]: + """An error object a reply holds, in one shape: its code, type and message, as given.""" + if not isinstance(e, dict): + return None + got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} + if e.get("message") is not None: + got["message"] = _cut(str(e["message"]), 500) + return got or {"message": _cut(json.dumps(e), 500)} + + +def _provider(got: dict) -> Optional[str]: + """The serving provider a reply (or a stream's event) names: its own + ``provider``, a Messages' or a Responses' object's, else the last attempt + of OpenRouter's routing facts (``openrouter_metadata``).""" + for obj in (got, got.get("message"), got.get("response")): + if isinstance(obj, dict) and isinstance(obj.get("provider"), str) and obj["provider"]: + return obj["provider"] + meta = got.get("openrouter_metadata") + attempts = meta.get("attempts") if isinstance(meta, dict) else None + named = [a.get("provider") for a in attempts or [] if isinstance(a, dict) and isinstance(a.get("provider"), str)] + return named[-1] if named else None + + +def _ending(fields: dict, served, end, error) -> dict: + for k, v in (("served_by", served), ("end", end), ("error", error)): + if v is not None: + fields[k] = v + return fields + + +def _whole(got: dict) -> tuple: + """(served_by, end, error) of a reply read whole.""" + served = _provider(got) + if "choices" in got: + first = (got.get("choices") or [{}])[0] or {} + return served, first.get("finish_reason"), _error(first.get("error")) or _error(got.get("error")) + if got.get("type") == "message": + return served, got.get("stop_reason"), None + if got.get("object") == "response" or "output" in got: + why = (got.get("incomplete_details") or {}).get("reason") if isinstance(got.get("incomplete_details"), dict) else None + status = got.get("status") + return served, f"{status}: {why}" if status and why else status, _error(got.get("error")) + return served, None, _error(got.get("error")) # an error alone: OpenRouter's, a Messages' {"type": "error"} + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: - """What a reply said: its text, its tool calls, its usage.""" + """What a reply said: its text, its tool calls, its usage; and, when it + says them, the serving provider that answered (``served_by``), how it + ended (``end``: ``finish_reason``, ``stop_reason`` or a Responses status) + and an error it holds though its status was a success (``error``).""" if not streamed: try: got = json.loads(body) except ValueError: return {} + if not isinstance(got, dict): + return {} if "choices" in got: m = ((got.get("choices") or [{}])[0].get("message")) or {} calls = [{"name": (c.get("function") or {}).get("name"), "arguments": _cut((c.get("function") or {}).get("arguments"), 2000)} for c in m.get("tool_calls") or []] - return {"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))} + return _ending({"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) if got.get("type") == "message" and isinstance(got.get("content"), list): text, calls = _text_of_messages(got) - return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))} + return _ending({"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) if got.get("object") == "response" or "output" in got: text, calls = _text_of_responses(got) - return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))} + return _ending({"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) if got.get("object") == "list" and path.split("?")[0].endswith("/embeddings"): # how many inputs, and the usage: never the vectors - return {"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))} - return {} + return _ending({"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))}, _provider(got), None, None) + return _ending({}, *_whole(got)) text, calls, usage, args = [], {}, None, {} + served = end = error = None told = None # Messages: the usage of message_start, then message_delta's on it for line in body.decode("utf-8", "replace").splitlines(): if not line.startswith("data:"): @@ -171,12 +237,21 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: ev = json.loads(data) except ValueError: continue + if not isinstance(ev, dict): + continue kind = ev.get("type") + served = served or _provider(ev) + if kind == "error": # Messages' error event, or a Responses one + error = _error(ev.get("error")) or _error(ev) + continue + if kind is None and isinstance(ev.get("error"), dict): # OpenRouter's error chunk inside a 200 + error = _error(ev["error"]) if kind == "message_start": told = dict(((ev.get("message") or {}).get("usage")) or {}) continue if kind == "message_delta": told = {**(told or {}), **{k: v for k, v in (ev.get("usage") or {}).items() if v is not None}} + end = (ev.get("delta") or {}).get("stop_reason") or end continue if kind == "content_block_start": b, i = ev.get("content_block") or {}, ev.get("index", 0) @@ -192,10 +267,14 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): - t, c = _text_of_responses(ev.get("response") or {}) - return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): + resp = ev.get("response") or {} + t, c = _text_of_responses(resp) + _, end, failed = _whole(resp) + return _ending({"text": t, "tool_calls": c, "usage": _usage(resp.get("usage"))}, served, end, failed or error) for ch in ev.get("choices") or []: + end = ch.get("finish_reason") or end + error = _error(ch.get("error")) or error d = ch.get("delta") or {} if d.get("content"): text.append(d["content"]) @@ -207,8 +286,8 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: args[i] = args.get(i, "") + (f.get("arguments") or "") if ev.get("usage"): usage = _usage(ev["usage"]) - return {"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)], - "usage": _usage(told) if told is not None else usage} + return _ending({"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)], + "usage": _usage(told) if told is not None else usage}, served, end, error) def _request_tail(req: dict) -> dict: @@ -355,7 +434,7 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, - ledger: Optional[str] = None, unit_caps: bool = True): + ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -373,8 +452,11 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s (:func:`request_fields`) - ``fidelity``'s, never a normal run's. ``ledger``: the file of the run's spending cap (``ledger``), shared by every relay of the job: a call is booked there before it is sent, - and not sent when it does not fit.""" + and not sent when it does not fit. ``serving``: by model, the serving + providers that may answer a request sent as that model (:func:`routing`), + put on every such request in place of the caller's own.""" self.fields = fields + self.serving = {m: list(s) for m, s in (serving or {}).items()} self.ledger = ledgers.Ledger(ledger) if ledger else None # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them self.unit_caps = unit_caps @@ -437,11 +519,14 @@ def _said(self, e: Exception) -> str: text = text.replace(k, "[REDACTED]") return redact(text)[:200] - def _headers(self, caller) -> Dict[str, str]: + def _headers(self, caller, facts: bool = False) -> Dict[str, str]: + """``facts``: a pinned Chat Completions request, whose reply names no + serving provider unless OpenRouter is asked for its routing facts.""" out = forwarded(caller) has = {k.lower() for k in out} # a name in any case is the same header: never send it twice for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ - (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"): + (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key") + \ + (("X-OpenRouter-Metadata", "enabled"),) * facts: if name.lower() not in has: out[name] = value if self.key_header == "x-api-key": @@ -465,6 +550,7 @@ def _forward(self, h: BaseHTTPRequestHandler): req = {} asked = req.get("model") if isinstance(req, dict) else None side = False + routing_asked, pinned = None, None if isinstance(req, dict) and req: if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag): req["model"] = self.embedding_models[tag] @@ -477,6 +563,10 @@ def _forward(self, h: BaseHTTPRequestHandler): req["model"] = self.model if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"): req.setdefault("stream_options", {})["include_usage"] = True + pinned = self.serving.get(req["model"]) if isinstance(req.get("model"), str) else None + if pinned: + routing_asked = req.get("provider") + req["provider"] = routing(pinned) raw = json.dumps(req).encode() booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and @@ -499,7 +589,8 @@ def _forward(self, h: BaseHTTPRequestHandler): conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed try: - conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers)) + facts = bool(pinned) and rest.split("?")[0].endswith("/chat/completions") + conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: status, kind = 502, "application/json" @@ -523,6 +614,10 @@ def record(body): "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True + if isinstance(req, dict) and req.get("provider") is not None: + rec["request"]["routing"] = req["provider"] # as sent: the pin's, or the caller's own + if pinned and routing_asked is not None: + rec["request"]["routing_asked"] = routing_asked if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None @@ -531,6 +626,10 @@ def record(body): cost, how = self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how + e = rec["reply"].get("error") + if e: # a success that holds an error: a failed call (failure) + rec["error"] = (f"HTTP 200 with an error{' ' + str(e['code']) if 'code' in e else ''}: " + + str(e.get("message") or e.get("type") or ""))[:300] else: rec["error"] = _said(status, body) if booking is not None: @@ -706,10 +805,12 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or calls[-1].get("status") == 200: + if not calls or (calls[-1].get("status") == 200 and not (calls[-1].get("reply") or {}).get("error")): return None last = calls[-1] s, said = last.get("status"), last.get("error") or "" + if s == 200: # an error inside a success: a rate limit when its code says so + s = 429 if last["reply"]["error"].get("code") in (429, "429") else None if s == 429: return "RateLimitError", said if s in (401, 403): @@ -725,7 +826,7 @@ def failure(calls: List[dict]) -> Optional[tuple]: def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME] - [--public URL] [--prices JSON] [--ledger FILE]``. + [--public URL] [--prices JSON] [--ledger FILE] [--served-by MODEL=SLUG[,SLUG...] ...]``. ``--base`` is the provider's endpoint, ``--route`` a route sent as another model (a benchmark's judge), ``--public`` the address ``url`` hands out. @@ -747,7 +848,16 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", + help="the serving providers that may answer a request sent as MODEL, in order") a = ap.parse_args(argv) + serving = {} + for given in a.served_by: + model, _, slugs = given.rpartition("=") + if not model or not slugs: + print(json.dumps({"error": f"--served-by {given!r}: MODEL=SLUG[,SLUG...]"}), flush=True) + return 2 + serving[model] = slugs.split(",") try: with open(a.key_file) as f: key = f.read().strip() @@ -759,7 +869,7 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 705e4d1..7c5dc0a 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -99,7 +99,8 @@ def driver_path() -> str: PROVIDERS = { "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, - "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"}, + # routing: one model is served by several providers, and a request may say which may answer (--served-by) + "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY", "routing": True}, "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } @@ -133,6 +134,34 @@ def provider_endpoint(provider: str, env=os.environ) -> tuple: return base, key +_SERVER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def served_by(given, model: str) -> Dict[str, List[str]]: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable: by model, the + serving providers that may answer it, in the order they are tried, and no + other (OpenRouter's slugs, as a model's page gives them: ``deepinfra``, + ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused when + a slug is empty, malformed or given twice, or a model is named twice.""" + out: Dict[str, List[str]] = {} + for g in given or []: + name, eq, listed = str(g).rpartition("=") + name = name.strip() if eq else model + slugs = [s.strip() for s in listed.split(",")] + if not name: + raise Refused(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") + bad = [s for s in slugs if not _SERVER.fullmatch(s)] + if bad: + raise Refused(f"--served-by {g!r}: {', '.join(repr(s) for s in bad)} is not a serving provider's slug " + "(as a model's page names it: deepinfra, deepinfra/fp8)") + if len(set(slugs)) != len(slugs): + raise Refused(f"--served-by {g!r} names a serving provider twice") + if name in out: + raise Refused(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") + out[name] = slugs + return out + + def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit" @@ -327,7 +356,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, - max_usd_per_unit: Optional[float] = None): + max_usd_per_unit: Optional[float] = None, served_by: Optional[Dict[str, List[str]]] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -361,7 +390,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas (``options`` in its definition: a search, a mode), the rest at their defaults; a value that needs an embedding model gets a relay of the run's own for it, one that needs something of this machine - (``host``) is refused without it.""" + (``host``) is refused without it. + ``served_by``: by model, the serving providers that may answer it + (:func:`served_by`), put on every request sent as that model by the + run's relays (``relay.routing``) and recorded in the job's config. + Refused when the provider routes no model to several (only + ``openrouter`` does), or when it names a model the run does not call.""" backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") @@ -478,6 +512,16 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) + self.serving = {m: list(s) for m, s in (served_by or {}).items()} + if self.serving: + if not PROVIDERS.get(provider, {}).get("routing"): + raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " + "among; the option needs a provider that routes a model to several (openrouter)") + called = self.models_called() + stray = [m for m in self.serving if m not in called] + if stray: + raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + + (", ".join(called) or "no model")) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -599,7 +643,7 @@ def start(self) -> str: image=self.setup_driver.image if self.setup_driver else self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, shard=shards.record(*self.shard, len(everything), units, self.limit) if self.shard else None, - repeat=self.repeat, prices=self.prices_record(), + repeat=self.repeat, prices=self.prices_record(), served_by=self.serving, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -822,6 +866,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False, + serving=self.serving, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -998,7 +1043,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), - environment=environment or "", + environment=environment or "", serving=getattr(self, "serving", None), image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" @@ -1207,7 +1252,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file()).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1236,7 +1281,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py new file mode 100644 index 0000000..9969c58 --- /dev/null +++ b/app/tests/test_standard_served_by.py @@ -0,0 +1,402 @@ +"""A pin of the serving providers that may answer a model (``--served-by``): +the relay puts OpenRouter's routing field on every request it sends as that +model and on no other, in place of the caller's own; each call records the +routing sent, the provider that served the reply, how the reply ended and an +error inside a success, which makes the call a failed one; the job and each +trial say who served their calls and whether the pin held; ``merge`` refuses +shards of different pins; a container's relay is given the pin as the unit's +own relay is. Without a pin a request goes on byte for byte as it did. Every +upstream here is a fake on this machine's loopback: no request leaves it.""" +from __future__ import annotations + +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +import urllib.request +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import progress +from nullius.standard import cli, container, definitions, install, job, merge, relay, run + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIN = {"order": ["prov-a", "prov-b/fp8"], "allow_fallbacks": False} + + +def sse(*events) -> bytes: + return b"".join(b"data: " + json.dumps(e).encode() + b"\n\n" for e in events) + b"data: [DONE]\n\n" + + +# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions, Messages, Responses. +CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}} +CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"delta": {"content": "h"}, "finish_reason": None}]}, + {"id": "c1", "provider": "Prov A", "choices": [{"delta": {"content": "i"}, "finish_reason": "length"}]}, + {"id": "c1", "provider": "Prov A", "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1}}) +MESSAGES = {"type": "message", "provider": "Prov B", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}} +MESSAGES_SSE = sse({"type": "message_start", "message": {"type": "message", "provider": "Prov B", "usage": {"input_tokens": 3}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 1}}, + {"type": "message_stop"}) +RESPONSES = {"object": "response", "provider": "Prov C", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}], + "usage": {"input_tokens": 3, "output_tokens": 1}} +RESPONSES_SSE = sse({"type": "response.created", "response": {"provider": "Prov C", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": {"provider": "Prov C", "status": "completed", "output": [], + "usage": {"input_tokens": 3, "output_tokens": 1}}}) +SHAPES = {("/chat/completions", False): (CHAT, "Prov A", "stop"), ("/chat/completions", True): (CHAT_SSE, "Prov A", "length"), + ("/messages", False): (MESSAGES, "Prov B", "end_turn"), ("/messages", True): (MESSAGES_SSE, "Prov B", "max_tokens"), + ("/responses", False): (RESPONSES, "Prov C", "incomplete: max_output_tokens"), + ("/responses", True): (RESPONSES_SSE, "Prov C", "completed")} + + +def body_for(path: str, stream: bool) -> dict: + """A caller's request of that shape.""" + if path == "/responses": + return {"model": "asked", "input": "hello", "stream": stream} + return {"model": "asked", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5, "stream": stream} + + +class Upstream(BaseHTTPRequestHandler): + """A provider on the loopback: keeps every request as it came, bytes and headers, and answers it in its shape; + ``Upstream.next`` replaces the next reply (status, content type, body).""" + seen = [] + next = None + + def log_message(self, *a): + pass + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + Upstream.seen.append((self.path, {k.lower(): v for k, v in self.headers.items()}, raw)) + req = json.loads(raw) + if Upstream.next is not None: + (status, kind, body), Upstream.next = Upstream.next, None + else: + path = "/" + self.path.split("/v1/", 1)[-1] + reply = SHAPES[(path, bool(req.get("stream")))][0] + status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ + reply if isinstance(reply, bytes) else json.dumps(reply).encode() + self.send_response(status) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url: str, body: dict) -> bytes: + req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json", + "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + with urllib.request.urlopen(req, timeout=10) as r: + return r.read() + + +class WithUpstream(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.tmp = tempfile.mkdtemp() + Upstream.seen, Upstream.next = [], None + self.relays = [] + + def tearDown(self): + for r in self.relays: + r.stop() + shutil.rmtree(self.tmp, ignore_errors=True) + + def relay(self, **kw): + r = relay.Relay(self.base, "or-test", os.path.join(self.tmp, "rec"), **kw).start() + self.relays.append(r) + return r + + def sent(self, i=-1) -> dict: + return json.loads(Upstream.seen[i][2]) + + +class TheRelay(WithUpstream): + def test_the_pin_goes_on_every_request_sent_as_the_pinned_model_and_on_no_other(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0-judge", model="m-judge") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0") + "/messages", body_for("/messages", True)) + self.assertEqual([(json.loads(raw)["model"], json.loads(raw).get("provider")) for _, _, raw in Upstream.seen], + [("m-run", PIN), ("m-judge", None), ("m-run", PIN)]) + self.assertEqual([c["request"].get("routing") for c in r.calls("u0")], [PIN, PIN]) + self.assertNotIn("routing", r.calls("u0-judge")[0]["request"]) + + def test_a_callers_own_routing_is_replaced_and_both_are_recorded(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"sort": "price"})) + self.assertEqual(self.sent()["provider"], PIN) + (c,) = r.calls("u0") + self.assertEqual((c["request"]["routing"], c["request"]["routing_asked"]), (PIN, {"sort": "price"})) + + def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self): + r = self.relay(model="m-run") + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"only": ["prov-z"]})) + self.assertEqual(self.sent()["provider"], {"only": ["prov-z"]}) + (c,) = r.calls("u0") + self.assertEqual(c["request"]["routing"], {"only": ["prov-z"]}) + self.assertNotIn("routing_asked", c["request"]) + + def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + for (path, stream), (_, who, end) in SHAPES.items(): + with self.subTest(path=path, stream=stream): + post(r.url("u0") + path, body_for(path, stream)) + self.assertEqual(self.sent()["provider"], {"order": ["prov-a"], "allow_fallbacks": False}) + reply = r.calls("u0")[-1]["reply"] + self.assertEqual((reply.get("served_by"), reply.get("end"), reply.get("error")), (who, end, None)) + self.assertEqual(reply["text"], "hi" if path != "/responses" or not stream else "") + + def test_an_error_inside_a_200_is_recorded_and_makes_the_call_a_failed_one(self): + r = self.relay(model="m-run") + errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", + {"code": 502, "message": "Provider returned error"}), + (sse({"id": "c1", "provider": "Prov A", "error": {"code": "server_error", "message": "upstream ended"}, + "choices": [{"delta": {}, "finish_reason": "error"}]}), "text/event-stream", + {"code": "server_error", "message": "upstream ended"}), + (sse({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), "text/event-stream", + {"type": "overloaded_error", "message": "Overloaded"})] + for body, kind, said in errors: + with self.subTest(said=said): + Upstream.next = (200, kind, body) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", kind == "text/event-stream")) + c = r.calls("u0")[-1] + self.assertEqual((c["status"], c["reply"]["error"]), (200, said)) + self.assertIn(said["message"], c["error"]) + self.assertEqual(relay.failure(r.calls("u0")), ("ApiError", c["error"])) + Upstream.next = (200, "application/json", b'{"error":{"code":429,"message":"Rate limit exceeded"}}') + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(relay.failure(r.calls("u0"))[0], "RateLimitError") + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) # answered: the unit recovered + self.assertIsNone(relay.failure(r.calls("u0"))) + + def test_a_pinned_chat_request_asks_for_the_routing_facts_and_reads_its_provider_there(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + Upstream.next = (200, "application/json", json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, + openrouter_metadata={"attempts": [{"provider": "Prov X", "status": 503}, + {"provider": "Prov A", "status": 200}]})).encode()) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled") + self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") + post(r.url("u0") + "/messages", body_for("/messages", False)) # its reply names its provider: nothing asked + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + plain = self.relay(model="m-run") + post(plain.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + + def test_without_a_pin_every_request_goes_on_byte_for_byte_as_before(self): + """What a relay sent before pins existed: the caller's body with the model set (and a streamed Chat + Completions request's usage flag), serialised once; a pin on another model changes none of it.""" + for serving in (None, {"m-other": ["prov-a"]}): + r = self.relay(model="m-run", serving=serving) + for (path, stream) in SHAPES: + for extra in ({}, {"provider": {"order": ["prov-z"]}}): + with self.subTest(serving=serving, path=path, stream=stream, extra=extra): + asked = dict(body_for(path, stream), **extra) + post(r.url("u0") + path, asked) + expected = dict(asked, model="m-run") + if stream and path == "/chat/completions": + expected.setdefault("stream_options", {})["include_usage"] = True + self.assertEqual(Upstream.seen[-1][2], json.dumps(expected).encode()) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + + +class TheSidecar(WithUpstream): + def setUp(self): + super().setUp() + self.enterContext(mock.patch.object(container, "_security", "")) # a default rootful engine, never asked + + def test_the_containers_relay_is_given_the_pin_on_its_command_line(self): + argv = container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", base="http://x/v1", + model="m-run", routes={"u0-judge": "m-judge"}, + serving={"m-run": ["prov-a", "prov-b/fp8"], "m-judge": ["prov-c"]}) + got = [argv[i + 1] for i, a in enumerate(argv) if a == "--served-by"] + self.assertEqual(got, ["m-judge=prov-c", "m-run=prov-a,prov-b/fp8"]) + self.assertNotIn("--served-by", container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", + base="http://x/v1", model="m-run", routes={})) + + def test_the_unit_hands_its_pin_to_the_sidecars_command(self): + unit = container.Unit("job", "u0", os.path.join(self.tmp, "rec"), image="img", app=APP, base=self.base, key="or-test", + model="m-run", serving={"m-run": ["prov-a"]}) + ran = [] + + def docker(args, **kw): + ran.append(args) + if "run" in args[:1]: + raise container.Unavailable("no engine in this test") + return subprocess.CompletedProcess(args, 0, "", "") + with mock.patch.object(container, "_docker", docker), self.assertRaises(container.Unavailable): + unit.start() + (started,) = [a for a in ran if a[:1] == ["run"]] + self.assertIn("m-run=prov-a", started[started.index("--served-by") + 1:]) + + def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + p = subprocess.Popen([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", "--base", self.base, + "--record", os.path.join(self.tmp, "rec"), "--key-file", key, "--model", "m-run", + "--served-by", "m-run=prov-a,prov-b/fp8"], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) + try: + url = json.loads(p.stdout.readline())["listening"] + post(url + "/u0/v1/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(self.sent()["provider"], PIN) + finally: + p.terminate() + p.wait(10) + p.stdout.close() + + +class TheRun(WithUpstream): + def setUp(self): + super().setUp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = definitions.get("benchmark", "fakebench", ROOT) + self.a = definitions.get("agent", "fake", ROOT) + + def tearDown(self): + install.HOME = self._home + super().tearDown() + + def test_the_option_reads_model_and_slugs_and_refuses_what_it_cannot_take(self): + self.assertEqual(run.served_by(["prov-a", "m-judge=prov-b/fp8, prov-c"], "m-run"), + {"m-run": ["prov-a"], "m-judge": ["prov-b/fp8", "prov-c"]}) + for given, said in ((["m-run=prov-a,"], "not a serving provider's slug"), (["m-run=a b"], "not a serving provider's slug"), + (["prov-a,prov-a"], "twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), + (["prov-a"], "MODEL=SLUG")): + with self.subTest(given=given), self.assertRaisesRegex(run.Refused, said): + run.served_by(given, "m-run" if given != ["prov-a"] else "") + + def test_a_provider_that_routes_no_model_to_several_is_refused_before_any_unit(self): + for provider, env in (("openai", {"OPENAI_API_KEY": "sk-x"}), ("anthropic", {"ANTHROPIC_API_KEY": "x"}), + ("custom", {"MODEL_API_URL": self.base})): + with self.subTest(provider=provider), self.assertRaisesRegex(run.Refused, "no serving providers to choose among"): + run.Run(self.b, self.a, model="m-1", provider=provider, env=env, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + + def test_a_model_the_run_does_not_call_is_refused_before_any_unit(self): + with self.assertRaisesRegex(run.Refused, "names m-other, which this run does not call: it calls m-1"): + run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + + def test_the_command_line_refuses_with_exit_2(self): + for patch in (mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_other", return_value=self.a), + mock.patch.object(cli, "_exit_on_term"), + mock.patch.object(cli.settings, "load"), mock.patch.object(progress, "ECHO", None)): + patch.start() + self.addCleanup(patch.stop) + for extra, said in ((["--provider", "openai", "--served-by", "prov-a"], "no serving providers"), + (["--provider", "openrouter", "--served-by", "m-2=prov-a"], "does not call"), + (["--provider", "openrouter", "--served-by", "m-1=bad slug"], "not a serving provider's slug")): + with self.subTest(extra=extra), mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-x", "OPENROUTER_API_KEY": "or-test"}): + out = io.StringIO() + with redirect_stdout(out): + code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--jobs", + os.path.join(self.tmp, "jobs"), *extra]) + self.assertEqual(code, 2) + self.assertTrue(out.getvalue().strip().splitlines()[-1].startswith("refused:"), out.getvalue()) + self.assertIn(said, out.getvalue()) + + def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + with mock.patch.dict(run.PROVIDERS, {"openrouter": dict(run.PROVIDERS["openrouter"], base=self.base)}): + job_dir = run.run(b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding", + served_by={"m-1": ["prov-a"]}) + self.assertTrue(Upstream.seen) + self.assertTrue(all(json.loads(raw).get("provider") == {"order": ["prov-a"], "allow_fallbacks": False} + for _, _, raw in Upstream.seen)) + with open(os.path.join(job_dir, "config.json")) as f: + cfg = json.load(f) + self.assertEqual(cfg["standard"]["served_by"], {"m-1": ["prov-a"]}) + trials = {d.split("__")[0]: json.load(open(os.path.join(job_dir, d, "result.json")))["standard"] + for d in os.listdir(job_dir) if "__u" in d} + n = len(Upstream.seen) + self.assertEqual(trials["model"]["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + self.assertEqual(trials["good"]["served"], {"calls": {}, "unnamed": 0, "held": {"m-1": None}}) # it called no model + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + self.assertTrue(all(c["request"]["routing"]["order"] == ["prov-a"] and c["reply"]["served_by"] == "Prov A" for c in calls)) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + + def test_the_containers_unit_is_given_the_runs_pin(self): + with mock.patch.object(container, "unusable", return_value=None): + r = run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, backend="container", + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + r.image, r.setup_driver, r.licences = "img", None, run.Licences(self.b, given={}) + trial = os.path.join(self.tmp, "trial") + with mock.patch.object(container, "Unit") as unit, mock.patch.object(run.Run, "unit_image", return_value="img"): + r.unit_relay("u0000", trial, {}) + self.assertEqual(unit.call_args.kwargs["serving"], {"m-1": ["prov-a"]}) + + +class TheRecord(unittest.TestCase): + def test_who_served_a_units_calls_and_whether_the_pin_held(self): + def call(model, who=None, status=200, error=None): + reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} + return {"model": model, "status": status, "reply": reply} + pin = {"m-1": ["google-ai-studio", "prov-b/fp8"], "m-judge": ["prov-c"]} + self.assertEqual(job.served([call("m-1", "Google AI Studio"), call("m-1", "Prov B"), call("m-1", status=429)], pin), + {"calls": {"Google AI Studio": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": True, "m-judge": None}}) + self.assertEqual(job.served([call("m-1", "Prov Z"), call("m-1")], pin)["held"]["m-1"], "unknown") + self.assertEqual(job.served([call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered + self.assertEqual(job.served([call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) + self.assertIsNone(job.served([call("m-1")], None)) + parts = [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 1, "held": {"m-1": True}}}}, + {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}, + {"standard": {}}] + self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) + + +class TheMerge(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing said"}}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_shards_of_different_pins_are_refused(self): + jobs = [run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=os.path.join(self.tmp, "jobs"), + env={}, say=lambda s: None, shard=(i, 2), launch="L", name=f"run-shard-{i}of2") for i in (1, 2)] + for job_dir, pin in zip(jobs, ({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]})): + path = os.path.join(job_dir, "config.json") + with open(path) as f: + cfg = json.load(f) + cfg["standard"]["served_by"] = pin + with open(path, "w") as f: + json.dump(cfg, f) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + self.assertEqual(merge.check(jobs, force=True)["differ"][0].split(":")[0], "serving providers") + + +if __name__ == "__main__": + unittest.main() From ebe1a1963fdaedebcf4deaa369886ec901c4d424 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 14:55:02 +0100 Subject: [PATCH 02/24] docs(standard): describe --served-by and the record of who served a call The run page gains the option and its rules: the slugs, what the relay sends, the refusals, the record and how a pin reads as held. The standard page names the record's new fields and the pin among merge's checks, and the providers page says that a run may choose OpenRouter's serving providers. --- docs/cli/run.md | 24 +++++++++++++++++++++++- docs/settings/model-providers.md | 2 +- docs/standard.md | 4 +++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index e5cf1d2..f95eb7a 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -5,7 +5,8 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--limit N] [-j N] \ [--unsandboxed | --backend container] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ - [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] \ + [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--served-by [MODEL=]SLUG[,SLUG...]]... \ + [--max-usd USD] [--max-usd-per-unit USD] \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -28,6 +29,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | | `--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | +| `--served-by [MODEL=]SLUG[,SLUG...]` | On OpenRouter, the serving providers that may answer a model (repeatable): [below](#serving-providers) | | `--max-usd USD`, `--max-usd-per-unit USD` | A spending cap for the run, for each unit: [below](#a-spending-cap) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | | `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run | @@ -64,6 +66,26 @@ nullius standard run locomo model --model gpt-4o-mini | From the builder | *Limits*: one cap shared in equal parts by the run's jobs, and one per unit | | Shards | A shard's cap is its own: [Shard and merge](shard-and-merge.md) | +## Serving providers + +On OpenRouter one model is served by several providers, which may differ in precision, speed and price. `--served-by deepinfra` lets only DeepInfra answer the run's model; `--served-by m=deepinfra/fp8,together` lets DeepInfra's fp8 endpoint answer the model `m`, then Together when it cannot. + +```bash title="One provider for the run's model" +nullius standard run humaneval reference --provider openrouter --model meta-llama/llama-3.3-70b-instruct --served-by deepinfra --unsandboxed +``` + +| Rule | Detail | +| --- | --- | +| A slug | As the model's page on OpenRouter names its provider: `deepinfra`, `deepinfra/fp8`. Several are tried in their order, and no other provider answers | +| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | +| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A Chat Completions request also asks for OpenRouter's routing facts (`X-OpenRouter-Metadata: enabled`), where its reply names the provider that served it. A model with no pin is sent as before | +| Refused before any unit | A provider other than `openrouter`; a model the run does not call; a slug that is empty, malformed or given twice; a model named twice | +| No provider can serve it | OpenRouter answers with an error and nothing else is tried; the call fails as any refused call does | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, `reply.end` and `reply.error` | +| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug (lowercased, its words joined by dashes: `Google AI Studio` is `google-ai-studio`), `unknown` when one names none or another name, `null` with no answered call | +| Shards | `merge` refuses shards whose pins differ, unless `--force` | +| The builder | A recipe has no field for it yet: a pinned run starts from the command line | +
- [The standard](../standard.md#a-run) diff --git a/docs/settings/model-providers.md b/docs/settings/model-providers.md index efcd70e..5c8362b 100644 --- a/docs/settings/model-providers.md +++ b/docs/settings/model-providers.md @@ -6,7 +6,7 @@ The key or endpoint a run's model calls go to. | --- | --- | --- | | TypeSafe AI | `TYPESAFE_API_KEY` | System One judgement only | | OpenAI | `OPENAI_API_KEY` | api.openai.com | -| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed | +| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed; a run may choose the serving providers of a model ([`--served-by`](../cli/run.md#serving-providers)) | | Anthropic | `ANTHROPIC_API_KEY` | Standard runs only, through the relay | | OpenAI-compatible | A name, a URL, an optional key, per endpoint | Ollama, vLLM, LM Studio or a hosted service, API path included | diff --git a/docs/standard.md b/docs/standard.md index 9d62ccb..47341a8 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -147,6 +147,8 @@ record); verifier/reward.txt and score.json | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`). A call with such an error is a failed call | +| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls by serving provider and, for each pinned model, `held` ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | ## Safety @@ -340,7 +342,7 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | Shards by the unit's id | `sha256(unit id) mod N` (`standard/shard.py`); changing N moves most units. Order: the driver's `units`, the shard, then `--limit` (`--shard 1/2 --limit 6` is the first six units of shard 1) | | A unit keeps its place | `u0042` is the 43rd unit of the whole dataset in every shard | | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model) | -| `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, backend or N are refused unless `--force`; two trials of one unit always | +| `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, serving providers, backend or N are refused unless `--force`; two trials of one unit always | | `cli prove --shard I/N` | The controls and the leak check on that shard's units; each record says `shard` | ```bash title="Two machines, one merge" From 76305acef108c3732444065b2a2e5b4524c64a99 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:02:09 +0100 Subject: [PATCH 03/24] docs(standard): state the serving providers' differences in a sentence of their own --- docs/cli/run.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index f95eb7a..a050165 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -68,7 +68,7 @@ nullius standard run locomo model --model gpt-4o-mini ## Serving providers -On OpenRouter one model is served by several providers, which may differ in precision, speed and price. `--served-by deepinfra` lets only DeepInfra answer the run's model; `--served-by m=deepinfra/fp8,together` lets DeepInfra's fp8 endpoint answer the model `m`, then Together when it cannot. +On OpenRouter one model is served by several providers. They may differ in precision, speed and price. `--served-by deepinfra` lets only DeepInfra answer the run's model; `--served-by m=deepinfra/fp8,together` lets DeepInfra's fp8 endpoint answer the model `m`, then Together when it cannot. ```bash title="One provider for the run's model" nullius standard run humaneval reference --provider openrouter --model meta-llama/llama-3.3-70b-instruct --served-by deepinfra --unsandboxed From f56bb3c6ba6abdb1cfebc9fa5d0b3b3d18910bcc Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:28:39 +0100 Subject: [PATCH 04/24] fix(standard): read who served a call apart from the record, and change no outcome without a pin A review found the pin's record could copy credentials, miss calls and change what a run without the option records and decides. Now: - The facts of a reply (who served it, how it ended, an error under a 200) are read by reply_facts, apart from read_reply, which is as before: the text, tool calls and usage a record held are read as they were, and no fact can raise into the delivery of a reply. Every choice is read for its end and its error, by index; a top-level error is read whatever else the reply holds; a completed Responses event's routing facts name who served it; a fact of the wrong type is left out. - An error under a 200 is recorded beside the reply and nothing else follows from it: the call keeps its status and failure reads it as before. - The recorded routing objects and facts are cleaned: a value under a credential's name, the relay's own key and key-shaped strings are removed. What is sent is unchanged. - The routing-facts header goes to OpenRouter alone, as enabled whatever the caller sent. - One check of a pin, shared by the run, its command line and the relay's own command; a control is refused a pin on a model it never calls. - Who served a unit counts its judge's and simulated user's routes, and the job the benchmark's own relay's calls; the tokens and the cost stay the agent's. A trial whose answered calls name no provider keeps its count. A name reads as a slug by its case and spaces alone. - merge compares pins as mappings, and a forced merge of different pins reads held unknown for such a model. --- app/nullius/standard/job.py | 34 ++- app/nullius/standard/merge.py | 18 +- app/nullius/standard/relay.py | 341 +++++++++++++++-------- app/nullius/standard/run.py | 53 ++-- app/tests/test_standard_served_by.py | 403 +++++++++++++++++++++------ 5 files changed, 615 insertions(+), 234 deletions(-) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 21d0211..ee91a09 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -402,8 +402,9 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: def _named(name: str) -> str: """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``Google AI Studio`` - reads ``google-ai-studio``). A run fetches no list of providers, so this is all a name is matched by.""" - return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + reads ``google-ai-studio``) and nothing else folded: ``Prov/A`` is not ``prov-a``. A run fetches no list of + providers, and this is all a name is matched by.""" + return re.sub(r"\s+", "-", name.strip().lower()) def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> Optional[dict]: @@ -411,7 +412,8 @@ def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> those that named none; and, for each model the run pinned (``serving``), whether the pin held - true when every answered call sent as that model names a provider that reads as one of its pinned slugs (``_named``, a slug's first part), ``unknown`` when one names none or another name, None when it has no answered call. - None for a unit whose calls name no provider in a run with no pin.""" + A slug's endpoint variant (``/fp8``) is not in a reply's name: ``held`` says which provider answered, and + cannot tell one of its endpoints from another. None for a unit with no answered call in a run with no pin.""" answered = [c for c in calls if c.get("status") == 200 and not (c.get("reply") or {}).get("error")] by: Dict[str, int] = {} unnamed = 0 @@ -421,7 +423,7 @@ def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> by[name] = by.get(name, 0) + 1 else: unnamed += 1 - if not by and not serving: + if not answered and not serving: return None out = {"calls": dict(sorted(by.items())), "unnamed": unnamed} if serving: @@ -435,9 +437,10 @@ def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> return out -def served_total(results: List[dict]) -> Optional[dict]: - """The job's ``served``: its trials' added up, a pin held where it held in every trial that called its model.""" - parts = [p for p in ((r.get("standard") or {}).get("served") for r in results) if p] +def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[dict]: + """The job's ``served``: its trials' added up with ``also`` (the benchmark's own relay's, :func:`served`), a + pin held where it held in every one that called its model.""" + parts = [p for p in [(r.get("standard") or {}).get("served") for r in results] + [also] if p] if not parts: return None calls: Dict[str, int] = {} @@ -458,10 +461,12 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), - serving: Optional[Dict[str, List[str]]] = None) -> dict: + serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None) -> dict: """Write one unit's trial and return its result. ``serving``: the run's pin of serving providers by model; ``standard.served`` says who served - the unit's calls and whether the pin held (:func:`served`). ``environment``: the + ``served_calls`` (the unit's calls on every route of its relay: its + agent's, its judge's, its simulated user's; ``calls`` when not given) and + whether the pin held (:func:`served`). ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began @@ -484,7 +489,7 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), **({"environment": environment} if environment else {}), **({"image": image} if image else {}), - **({"served": who} if (who := served(calls, serving)) else {})}} + **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score @@ -599,14 +604,15 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} -def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): +def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None, + served: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, what was spent, whether it stopped the run, the units it cut off and - those never begun. ``served``: who served its trials' calls, when a - trial says it (:func:`served`).""" + those never begun. ``served``: who served the benchmark's own relay's + calls (:func:`served`); the job's ``served`` adds its trials' to them.""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) - who = served_total(results) + who = served_total(results, served) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 9f23261..010224d 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -65,6 +65,20 @@ def _at(d, path): return d +def _served(shards: List[dict], results: List[dict]) -> Optional[dict]: + """The merged job's ``served`` (``job.served_total``). A model the shards pinned differently - merged + with ``force`` - reads ``unknown``: the merged config keeps the first shard's pin, which the others' calls + were never held to.""" + who = job.served_total(results) + pins = [_at(s["config"], ("standard", "served_by")) or {} for s in shards] + pins = [p if isinstance(p, dict) else {} for p in pins] + differ = sorted(m for m in {m for p in pins for m in p} if len({json.dumps(p.get(m)) for p in pins}) > 1) + if differ: + who = who or {"calls": {}, "unnamed": 0} + who["held"] = dict(who.get("held") or {}, **{m: "unknown" for m in differ}) + return who + + def _trials(job_dir: str) -> List[str]: return sorted(n for n in os.listdir(job_dir) if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json"))) @@ -107,7 +121,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: raise Refused("a job is named twice") differ = [] for label, path in SAME: - seen = {json.dumps(_at(s["config"], path)) for s in shards} + seen = {json.dumps(_at(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content if len(seen) > 1: differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards)) if differ and not force: @@ -223,7 +237,7 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), "scored": len(scores), "mean_score": job.mean_score(results)}, **({"budget": budget} if budget else {}), - **({"served": who} if (who := job.served_total(results)) else {})}, f, indent=1) + **({"served": who} if (who := _served(shards, results)) else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 9c4f139..231b184 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -8,11 +8,12 @@ A reply is passed through as it arrives - a streamed one (server-sent events) byte for byte - and recorded once it is complete, one line per call in ``/.jsonl``: when it started and ended, the model, the status, -the tokens, and an abridged request and reply: the reply's serving provider -when it names one (``served_by``), how it ended (``end``) and an error it -holds inside a success (``error``), which makes the call a failed one. Three -API shapes are read, Chat Completions, Responses and Anthropic's Messages, -streamed or not. The +the tokens, and an abridged request and reply. Three API shapes are read, +Chat Completions, Responses and Anthropic's Messages, streamed or not. Read +apart from the rest (:func:`reply_facts`), the reply also says, when it does, +the serving provider that answered (``served_by``), how it ended (``end``) +and an error it holds though its status was a success (``error``); reading +them never changes what the caller receives, nor the rest of the record. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the user's price list, ``nullius.prices``), marked as an estimate. @@ -30,8 +31,8 @@ A run may pin, for a model, the serving providers that may answer it (``serving``, OpenRouter's routing): the relay then puts ``provider: {order, allow_fallbacks: false}`` on every request it sends as -that model, in place of the caller's own, and records both. Without a pin a -request's body goes on as it did. +that model, in place of the caller's own, and records both, cleaned of +credentials. Without a pin a request's body goes on as it did. One answer is the relay's own: a run with a spending cap (``ledger``) books each call before it goes out, and a call that does not fit is not sent. Its @@ -147,85 +148,28 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls -def routing(slugs) -> dict: - """OpenRouter's routing field for a strict pin: these serving providers, - tried in this order, and no other.""" - return {"order": list(slugs), "allow_fallbacks": False} - - -def _error(e) -> Optional[dict]: - """An error object a reply holds, in one shape: its code, type and message, as given.""" - if not isinstance(e, dict): - return None - got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} - if e.get("message") is not None: - got["message"] = _cut(str(e["message"]), 500) - return got or {"message": _cut(json.dumps(e), 500)} - - -def _provider(got: dict) -> Optional[str]: - """The serving provider a reply (or a stream's event) names: its own - ``provider``, a Messages' or a Responses' object's, else the last attempt - of OpenRouter's routing facts (``openrouter_metadata``).""" - for obj in (got, got.get("message"), got.get("response")): - if isinstance(obj, dict) and isinstance(obj.get("provider"), str) and obj["provider"]: - return obj["provider"] - meta = got.get("openrouter_metadata") - attempts = meta.get("attempts") if isinstance(meta, dict) else None - named = [a.get("provider") for a in attempts or [] if isinstance(a, dict) and isinstance(a.get("provider"), str)] - return named[-1] if named else None - - -def _ending(fields: dict, served, end, error) -> dict: - for k, v in (("served_by", served), ("end", end), ("error", error)): - if v is not None: - fields[k] = v - return fields - - -def _whole(got: dict) -> tuple: - """(served_by, end, error) of a reply read whole.""" - served = _provider(got) - if "choices" in got: - first = (got.get("choices") or [{}])[0] or {} - return served, first.get("finish_reason"), _error(first.get("error")) or _error(got.get("error")) - if got.get("type") == "message": - return served, got.get("stop_reason"), None - if got.get("object") == "response" or "output" in got: - why = (got.get("incomplete_details") or {}).get("reason") if isinstance(got.get("incomplete_details"), dict) else None - status = got.get("status") - return served, f"{status}: {why}" if status and why else status, _error(got.get("error")) - return served, None, _error(got.get("error")) # an error alone: OpenRouter's, a Messages' {"type": "error"} - - def read_reply(path: str, body: bytes, streamed: bool) -> dict: - """What a reply said: its text, its tool calls, its usage; and, when it - says them, the serving provider that answered (``served_by``), how it - ended (``end``: ``finish_reason``, ``stop_reason`` or a Responses status) - and an error it holds though its status was a success (``error``).""" + """What a reply said: its text, its tool calls, its usage.""" if not streamed: try: got = json.loads(body) except ValueError: return {} - if not isinstance(got, dict): - return {} if "choices" in got: m = ((got.get("choices") or [{}])[0].get("message")) or {} calls = [{"name": (c.get("function") or {}).get("name"), "arguments": _cut((c.get("function") or {}).get("arguments"), 2000)} for c in m.get("tool_calls") or []] - return _ending({"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) + return {"text": m.get("content"), "tool_calls": calls, "usage": _usage(got.get("usage"))} if got.get("type") == "message" and isinstance(got.get("content"), list): text, calls = _text_of_messages(got) - return _ending({"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) + return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))} if got.get("object") == "response" or "output" in got: text, calls = _text_of_responses(got) - return _ending({"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))}, *_whole(got)) + return {"text": text, "tool_calls": calls, "usage": _usage(got.get("usage"))} if got.get("object") == "list" and path.split("?")[0].endswith("/embeddings"): # how many inputs, and the usage: never the vectors - return _ending({"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))}, _provider(got), None, None) - return _ending({}, *_whole(got)) + return {"embeddings": len(got.get("data") or []), "usage": _usage(got.get("usage"))} + return {} text, calls, usage, args = [], {}, None, {} - served = end = error = None told = None # Messages: the usage of message_start, then message_delta's on it for line in body.decode("utf-8", "replace").splitlines(): if not line.startswith("data:"): @@ -237,21 +181,12 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: ev = json.loads(data) except ValueError: continue - if not isinstance(ev, dict): - continue kind = ev.get("type") - served = served or _provider(ev) - if kind == "error": # Messages' error event, or a Responses one - error = _error(ev.get("error")) or _error(ev) - continue - if kind is None and isinstance(ev.get("error"), dict): # OpenRouter's error chunk inside a 200 - error = _error(ev["error"]) if kind == "message_start": told = dict(((ev.get("message") or {}).get("usage")) or {}) continue if kind == "message_delta": told = {**(told or {}), **{k: v for k, v in (ev.get("usage") or {}).items() if v is not None}} - end = (ev.get("delta") or {}).get("stop_reason") or end continue if kind == "content_block_start": b, i = ev.get("content_block") or {}, ev.get("index", 0) @@ -267,14 +202,10 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): - resp = ev.get("response") or {} - t, c = _text_of_responses(resp) - _, end, failed = _whole(resp) - return _ending({"text": t, "tool_calls": c, "usage": _usage(resp.get("usage"))}, served, end, failed or error) + if kind in ("response.completed", "response.done", "response.incomplete"): + t, c = _text_of_responses(ev.get("response") or {}) + return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: - end = ch.get("finish_reason") or end - error = _error(ch.get("error")) or error d = ch.get("delta") or {} if d.get("content"): text.append(d["content"]) @@ -286,8 +217,188 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: args[i] = args.get(i, "") + (f.get("arguments") or "") if ev.get("usage"): usage = _usage(ev["usage"]) - return _ending({"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)], - "usage": _usage(told) if told is not None else usage}, served, end, error) + return {"text": "".join(text), "tool_calls": [{"name": calls[i], "arguments": _cut(args.get(i, ""), 2000)} for i in sorted(calls)], + "usage": _usage(told) if told is not None else usage} + + +# -- serving providers: the pin a run sets, and what a reply says of who served it -- +SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def routing(slugs) -> dict: + """OpenRouter's routing field for a strict pin: these serving providers, + tried in this order, and no other.""" + return {"order": list(slugs), "allow_fallbacks": False} + + +def serving_problems(serving) -> List[str]: + """What is wrong with a pin by model (``{model: [slug, ...]}``): the one + check the run, its command line and the relay's own command share.""" + if not isinstance(serving, dict): + return ["the serving providers are given by model: {MODEL: [SLUG, ...]}"] + bad = [] + for model, slugs in serving.items(): + if not isinstance(model, str) or not model.strip(): + bad.append("a pin names no model") + continue + if not isinstance(slugs, (list, tuple)) or not slugs: + bad.append(f"{model}: no serving provider") + continue + wrong = [s for s in slugs if not isinstance(s, str) or not SLUG.fullmatch(s)] + if wrong: + bad.append(f"{model}: {', '.join(repr(s) for s in wrong)} is not a serving provider's slug " + "(as a model's page names it: deepinfra, deepinfra/fp8)") + elif len(set(slugs)) != len(slugs): + bad.append(f"{model}: a serving provider is named twice") + return bad + + +def parse_served_by(given, model: str) -> tuple: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable, as ``({model: [slug, ...]}, problems)``. Without + ``MODEL=`` it is ``model``, the run's. A model named twice is a problem; so is each of :func:`serving_problems`.""" + out: Dict[str, List[str]] = {} + bad = [] + for g in given or []: + name, eq, listed = str(g).rpartition("=") + name = name.strip() if eq else (model or "") + if not name: + bad.append(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") + continue + if name in out: + bad.append(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") + continue + out[name] = [s.strip() for s in listed.split(",")] + return out, bad + serving_problems(out) + + +def _clean(value, secrets=()): + """A recorded copy, never what is sent: a value under a credential's name + replaced, the relay's own key cut out of every string, then ``redact``.""" + if isinstance(value, dict): + return {k: "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) for k, v in value.items()} + if isinstance(value, list): + return [_clean(v, secrets) for v in value] + if isinstance(value, str): + for s in secrets: + if s: + value = value.replace(s, "[REDACTED]") + return redact(value) + return value + + +def _name(v) -> Optional[str]: + return v if isinstance(v, str) and v else None + + +def _an_error(e) -> Optional[dict]: + """An error object, in one shape: its code, type and message, as given.""" + if not isinstance(e, dict) or not e: + return None + got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} + if isinstance(e.get("message"), str): + got["message"] = _cut(e["message"], 500) + return got or {"message": _cut(json.dumps(e), 500)} + + +def _provider(obj) -> Optional[str]: + """The serving provider an object of a reply names: its own ``provider``, + its ``message``'s or its ``response``'s, else the last attempt of + OpenRouter's routing facts (``openrouter_metadata``) there.""" + if not isinstance(obj, dict): + return None + inner = [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)] + for o in inner: + if _name(o.get("provider")): + return o["provider"] + for o in inner: + meta = o.get("openrouter_metadata") + attempts = meta.get("attempts") if isinstance(meta, dict) else None + named = [a["provider"] for a in attempts if isinstance(a, dict) and _name(a.get("provider"))] if isinstance(attempts, list) else [] + if named: + return named[-1] + return None + + +def _choices(choices, ends: dict, errors: dict) -> None: + """Each choice's end and error, by its index.""" + if not isinstance(choices, list): + return + for n, ch in enumerate(choices): + if not isinstance(ch, dict): + continue + i = ch.get("index") if isinstance(ch.get("index"), int) and not isinstance(ch.get("index"), bool) else n + if _name(ch.get("finish_reason")): + ends[i] = ch["finish_reason"] + e = _an_error(ch.get("error")) + if e and i not in errors: + errors[i] = e + + +def _status(resp) -> Optional[str]: + """A Responses object's end: its status, and why when it is incomplete.""" + if not isinstance(resp, dict) or not _name(resp.get("status")): + return None + why = resp.get("incomplete_details") + why = _name(why.get("reason")) if isinstance(why, dict) else None + return f"{resp['status']}: {why}" if why else resp["status"] + + +def _facts(path: str, body: bytes, streamed: bool) -> dict: + served, end, error, ends, errors = None, None, None, {}, {} + if not streamed: + got = json.loads(body) + if not isinstance(got, dict): + return {} + served, error = _provider(got), _an_error(got.get("error")) + _choices(got.get("choices"), ends, errors) + if got.get("type") == "message": + end = _name(got.get("stop_reason")) + elif got.get("object") == "response" or "output" in got: + end = _status(got) + else: + for line in body.decode("utf-8", "replace").splitlines(): + if not line.startswith("data:"): + continue + try: + ev = json.loads(line[5:].strip()) + except ValueError: + continue + if not isinstance(ev, dict): + continue + served = served or _provider(ev) + kind = ev.get("type") + if kind == "error": # Messages' error event, or a Responses one + error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) + elif isinstance(ev.get("error"), dict): # an error chunk under a 200 + error = error or _an_error(ev["error"]) + if kind == "message_delta" and isinstance(ev.get("delta"), dict): + end = _name(ev["delta"].get("stop_reason")) or end + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): + resp = ev.get("response") + end = _status(resp) or end + error = error or _an_error(resp.get("error") if isinstance(resp, dict) else None) + _choices(ev.get("choices"), ends, errors) + if ends: + end = ends[min(ends)] if len(ends) == 1 else {str(i): ends[i] for i in sorted(ends)} + if error is None and errors: + i = min(errors) + error = dict(errors[i], choice=i) + return {k: v for k, v in (("served_by", served), ("end", end), ("error", error)) if v is not None} + + +def reply_facts(path: str, body: bytes, streamed: bool) -> dict: + """What a reply says besides what :func:`read_reply` reads, read apart from + it: the serving provider that answered (``served_by``), how it ended + (``end``: ``finish_reason``, ``stop_reason`` or a Responses status; one per + choice, by index, when there are several) and an error it holds though its + status was a success (``error``: code, type and message; a top-level one + whatever else the reply holds, else a choice's). A fact of the wrong type + is left out, and a reply that cannot be read gives none: nothing here can + stop a reply reaching its caller.""" + try: + return _facts(path, body, streamed) + except Exception: + return {} def _request_tail(req: dict) -> dict: @@ -434,7 +545,8 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, - ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None): + ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, + facts: Optional[bool] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -454,9 +566,14 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s by every relay of the job: a call is booked there before it is sent, and not sent when it does not fit. ``serving``: by model, the serving providers that may answer a request sent as that model (:func:`routing`), - put on every such request in place of the caller's own.""" + put on every such request in place of the caller's own. ``facts``: a + pinned Chat Completions request asks OpenRouter for its routing facts, + where its reply names who served it; by default when ``base`` is + OpenRouter's.""" self.fields = fields self.serving = {m: list(s) for m, s in (serving or {}).items()} + host = (urllib.parse.urlsplit(base).hostname or "").lower() + self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) self.ledger = ledgers.Ledger(ledger) if ledger else None # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them self.unit_caps = unit_caps @@ -520,13 +637,15 @@ def _said(self, e: Exception) -> str: return redact(text)[:200] def _headers(self, caller, facts: bool = False) -> Dict[str, str]: - """``facts``: a pinned Chat Completions request, whose reply names no - serving provider unless OpenRouter is asked for its routing facts.""" + """``facts``: a pinned Chat Completions request to OpenRouter, sent + asking for its routing facts whatever the caller said of them.""" out = forwarded(caller) + if facts: + out = {k: v for k, v in out.items() if k.lower() != "x-openrouter-metadata"} + out["X-OpenRouter-Metadata"] = "enabled" has = {k.lower() for k in out} # a name in any case is the same header: never send it twice for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ - (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key") + \ - (("X-OpenRouter-Metadata", "enabled"),) * facts: + (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"): if name.lower() not in has: out[name] = value if self.key_header == "x-api-key": @@ -589,7 +708,7 @@ def _forward(self, h: BaseHTTPRequestHandler): conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed try: - facts = bool(pinned) and rest.split("?")[0].endswith("/chat/completions") + facts = bool(pinned) and self.facts and rest.split("?")[0].endswith("/chat/completions") conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: @@ -615,21 +734,24 @@ def record(body): if side: rec["side"] = True if isinstance(req, dict) and req.get("provider") is not None: - rec["request"]["routing"] = req["provider"] # as sent: the pin's, or the caller's own + # the routing as sent, and the caller's own the pin replaced: recorded copies, cleaned + rec["request"]["routing"] = _clean(req["provider"], (self.key, self.key.strip())) if pinned and routing_asked is not None: - rec["request"]["routing_asked"] = routing_asked + rec["request"]["routing_asked"] = _clean(routing_asked, (self.key, self.key.strip())) if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None if status == 200: rec["reply"] = read_reply(rest, body, streamed) + # who served it, how it ended, an error under the 200: beside what was read, never in its place + try: + rec["reply"].update({k: v for k, v in _clean(reply_facts(rest, body, streamed), (self.key, self.key.strip())).items() + if k not in rec["reply"]}) + except Exception: + pass cost, how = self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - e = rec["reply"].get("error") - if e: # a success that holds an error: a failed call (failure) - rec["error"] = (f"HTTP 200 with an error{' ' + str(e['code']) if 'code' in e else ''}: " - + str(e.get("message") or e.get("type") or ""))[:300] else: rec["error"] = _said(status, body) if booking is not None: @@ -805,12 +927,10 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or (calls[-1].get("status") == 200 and not (calls[-1].get("reply") or {}).get("error")): + if not calls or calls[-1].get("status") == 200: return None last = calls[-1] s, said = last.get("status"), last.get("error") or "" - if s == 200: # an error inside a success: a rate limit when its code says so - s = 429 if last["reply"]["error"].get("code") in (429, "429") else None if s == 429: return "RateLimitError", said if s in (401, 403): @@ -849,15 +969,12 @@ def serve(argv=None) -> int: ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", - help="the serving providers that may answer a request sent as MODEL, in order") + help="the serving providers that may answer a request sent as MODEL, in order, and no other") a = ap.parse_args(argv) - serving = {} - for given in a.served_by: - model, _, slugs = given.rpartition("=") - if not model or not slugs: - print(json.dumps({"error": f"--served-by {given!r}: MODEL=SLUG[,SLUG...]"}), flush=True) - return 2 - serving[model] = slugs.split(",") + serving, bad = parse_served_by(a.served_by, "") + if bad: + print(json.dumps({"error": "; ".join(bad)}), flush=True) + return 2 try: with open(a.key_file) as f: key = f.read().strip() diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 7c5dc0a..6ce474c 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -134,31 +134,16 @@ def provider_endpoint(provider: str, env=os.environ) -> tuple: return base, key -_SERVER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") - - def served_by(given, model: str) -> Dict[str, List[str]]: """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable: by model, the serving providers that may answer it, in the order they are tried, and no other (OpenRouter's slugs, as a model's page gives them: ``deepinfra``, - ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused when - a slug is empty, malformed or given twice, or a model is named twice.""" - out: Dict[str, List[str]] = {} - for g in given or []: - name, eq, listed = str(g).rpartition("=") - name = name.strip() if eq else model - slugs = [s.strip() for s in listed.split(",")] - if not name: - raise Refused(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") - bad = [s for s in slugs if not _SERVER.fullmatch(s)] - if bad: - raise Refused(f"--served-by {g!r}: {', '.join(repr(s) for s in bad)} is not a serving provider's slug " - "(as a model's page names it: deepinfra, deepinfra/fp8)") - if len(set(slugs)) != len(slugs): - raise Refused(f"--served-by {g!r} names a serving provider twice") - if name in out: - raise Refused(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") - out[name] = slugs + ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused as + ``relay.parse_served_by`` finds it wrong: the check the run and the + relay's own command make too.""" + out, bad = relay.parse_served_by(given, model) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) return out @@ -512,12 +497,18 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) - self.serving = {m: list(s) for m, s in (served_by or {}).items()} + self.serving = dict(served_by or {}) if self.serving: + bad = relay.serving_problems(self.serving) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + self.serving = {m: list(s) for m, s in self.serving.items()} if not PROVIDERS.get(provider, {}).get("routing"): raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " "among; the option needs a provider that routes a model to several (openrouter)") - called = self.models_called() + # a control calls no model of its own: only its judge, simulated user or an option's models + called = [m for m in self.models_called() + if other.get("_kind") != "control" or m in (self.judge_model, self.simulator_model, self.embed_model, self.rerank_model)] stray = [m for m in self.serving if m not in called] if stray: raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " @@ -716,7 +707,8 @@ def work_unit(iur): self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) self.capped = bool(budget and budget["stopped"]) - job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget) + job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget, + served=job.served(self.benchmark_calls(), getattr(self, "serving", None))) errored = sum(1 for x in results if x.get("exception_info")) self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results)) # the units' stage says why it ended early: a line of its own would be a stage that never starts @@ -1044,6 +1036,9 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), environment=environment or "", serving=getattr(self, "serving", None), + # who served the unit: its agent's calls and its judge's and simulated user's routes + served_calls=calls + ([c for side in ("judge", "simulator") for c in rl.calls(f"{tag}-{side}")] + if rl is not None else []), image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" @@ -1228,6 +1223,16 @@ def ledger_file(self) -> Optional[str]: led = getattr(self, "ledger", None) return led.path if led is not None else None + def benchmark_calls(self) -> List[dict]: + """The calls of the benchmark's own relay (an option's embeddings or reranking), every route of it.""" + folder = os.path.join(self.job_dir, ".work", "relay-benchmark") + calls = [] + for name in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + if name.endswith(".jsonl"): + with open(os.path.join(folder, name)) as f: + calls += [json.loads(line) for line in f if line.strip()] + return calls + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index 9969c58..8a06141 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -2,16 +2,19 @@ the relay puts OpenRouter's routing field on every request it sends as that model and on no other, in place of the caller's own; each call records the routing sent, the provider that served the reply, how the reply ended and an -error inside a success, which makes the call a failed one; the job and each -trial say who served their calls and whether the pin held; ``merge`` refuses -shards of different pins; a container's relay is given the pin as the unit's -own relay is. Without a pin a request goes on byte for byte as it did. Every +error inside a success, as cleaned copies, read apart from the rest of the +record; the job and each trial say who served the calls of every route and +whether the pin held; ``merge`` refuses shards of different pins; a +container's relay is given the pin as the unit's own relay is. A run without +the option sends the same requests and ends its units as it did. Every upstream here is a fake on this machine's loopback: no request leaves it.""" from __future__ import annotations +import contextlib import io import json import os +import select import shutil import subprocess import sys @@ -29,6 +32,8 @@ ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PIN = {"order": ["prov-a", "prov-b/fp8"], "allow_fallbacks": False} +KEY = "or-test-relay-key-0123456789" +NEW = ("served_by", "end", "error") # what the record's reply gains, beside what it held def sse(*events) -> bytes: @@ -38,8 +43,8 @@ def sse(*events) -> bytes: # Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions, Messages, Responses. CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 3, "completion_tokens": 1}} -CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"delta": {"content": "h"}, "finish_reason": None}]}, - {"id": "c1", "provider": "Prov A", "choices": [{"delta": {"content": "i"}, "finish_reason": "length"}]}, +CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "h"}, "finish_reason": None}]}, + {"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "i"}, "finish_reason": "length"}]}, {"id": "c1", "provider": "Prov A", "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1}}) MESSAGES = {"type": "message", "provider": "Prov B", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", "usage": {"input_tokens": 3, "output_tokens": 1}} @@ -67,11 +72,16 @@ def body_for(path: str, stream: bool) -> dict: return {"model": "asked", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5, "stream": stream} +def without_new(reply: dict) -> dict: + return {k: v for k, v in reply.items() if k not in NEW} + + class Upstream(BaseHTTPRequestHandler): """A provider on the loopback: keeps every request as it came, bytes and headers, and answers it in its shape; - ``Upstream.next`` replaces the next reply (status, content type, body).""" + ``Upstream.next`` replaces the next reply (status, content type, body), ``Upstream.always`` every one.""" seen = [] next = None + always = None def log_message(self, *a): pass @@ -80,11 +90,11 @@ def do_POST(self): raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) Upstream.seen.append((self.path, {k.lower(): v for k, v in self.headers.items()}, raw)) req = json.loads(raw) - if Upstream.next is not None: - (status, kind, body), Upstream.next = Upstream.next, None + if Upstream.next is not None or Upstream.always is not None: + (status, kind, body), Upstream.next = Upstream.next or Upstream.always, None else: path = "/" + self.path.split("/v1/", 1)[-1] - reply = SHAPES[(path, bool(req.get("stream")))][0] + reply = SHAPES.get((path, bool(req.get("stream"))), (CHAT,))[0] status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ reply if isinstance(reply, bytes) else json.dumps(reply).encode() self.send_response(status) @@ -94,9 +104,10 @@ def do_POST(self): self.wfile.write(body) -def post(url: str, body: dict) -> bytes: - req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json", - "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) +def post(url: str, body: dict, headers=None) -> bytes: + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY, + **(headers or {})}) with urllib.request.urlopen(req, timeout=10) as r: return r.read() @@ -115,8 +126,9 @@ def tearDownClass(cls): def setUp(self): self.tmp = tempfile.mkdtemp() - Upstream.seen, Upstream.next = [], None + Upstream.seen, Upstream.next, Upstream.always = [], None, None self.relays = [] + self.addCleanup(setattr, Upstream, "always", None) def tearDown(self): for r in self.relays: @@ -124,7 +136,7 @@ def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) def relay(self, **kw): - r = relay.Relay(self.base, "or-test", os.path.join(self.tmp, "rec"), **kw).start() + r = relay.Relay(self.base, kw.pop("key", "or-test"), os.path.join(self.tmp, f"rec{len(self.relays)}"), **kw).start() self.relays.append(r) return r @@ -158,6 +170,24 @@ def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self self.assertEqual(c["request"]["routing"], {"only": ["prov-z"]}) self.assertNotIn("routing_asked", c["request"]) + def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): + """An error that echoes the request's credentials, and a routing object holding a token: the record keeps + neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" + Upstream.always = (200, "application/json", json.dumps({"error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + for serving in (None, {"m-run": ["prov-a"]}): + with self.subTest(serving=serving): + r = self.relay(model="m-run", key=KEY, serving=serving) + asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}"}) + post(r.url("u0") + "/chat/completions", asked) + c = r.calls("u0")[-1] + self.assertNotIn(KEY, json.dumps(c)) + self.assertEqual(c["reply"]["error"]["code"], 401) + if serving is None: # sent as the caller sent it: the record alone is cleaned + self.assertEqual(self.sent()["provider"], asked["provider"]) + self.assertEqual(c["request"]["routing"]["token"], "[REDACTED]") + else: + self.assertEqual(c["request"]["routing_asked"]["token"], "[REDACTED]") + def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) for (path, stream), (_, who, end) in SHAPES.items(): @@ -166,60 +196,127 @@ def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): self.assertEqual(self.sent()["provider"], {"order": ["prov-a"], "allow_fallbacks": False}) reply = r.calls("u0")[-1]["reply"] self.assertEqual((reply.get("served_by"), reply.get("end"), reply.get("error")), (who, end, None)) - self.assertEqual(reply["text"], "hi" if path != "/responses" or not stream else "") - - def test_an_error_inside_a_200_is_recorded_and_makes_the_call_a_failed_one(self): + # what was read before is read as before, beside the new facts + self.assertEqual(without_new(reply), relay.read_reply(path, SHAPES[(path, stream)][0] if stream + else json.dumps(SHAPES[(path, stream)][0]).encode(), stream)) + + def test_several_choices_keep_their_endings_and_errors_by_index(self): + body = {"choices": [{"index": 0, "message": {"content": "ok"}, "finish_reason": "stop"}, + {"index": 1, "message": {}, "finish_reason": "error", "error": {"code": 429, "message": "limited"}}]} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(body).encode(), False), + {"end": {"0": "stop", "1": "error"}, "error": {"code": 429, "message": "limited", "choice": 1}}) + stream = sse({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}, {"index": 1, "delta": {}, "finish_reason": None}]}, + {"choices": [{"index": 1, "delta": {}, "finish_reason": "stop"}]}) + self.assertEqual(relay.reply_facts("/chat/completions", stream, True)["end"], {"0": "length", "1": "stop"}) + + def test_an_error_beside_a_whole_reply_is_read_in_every_shape(self): + error = {"code": 502, "message": "failed"} + for path, body in (("/messages", {"type": "message", "content": [], "error": error}), + ("/embeddings", {"object": "list", "data": [], "error": error}), + ("/chat/completions", {"choices": [{"message": {"content": "ok"}}], "error": error}), + ("/responses", {"object": "response", "status": "failed", "output": [], "error": error})): + with self.subTest(path=path): + self.assertEqual(relay.reply_facts(path, json.dumps(body).encode(), False)["error"], error) + + def test_a_completed_responses_events_routing_facts_name_who_served_it(self): + body = sse({"type": "response.completed", "response": {"object": "response", "status": "completed", "output": [], + "openrouter_metadata": {"attempts": [{"provider": "Prov A", "status": 200}]}}}) + self.assertEqual(relay.reply_facts("/responses", body, True), {"served_by": "Prov A", "end": "completed"}) + + def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): + odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"attempts": 1}}, + {"type": "message", "content": [], "provider": 7, "error": "a string", "stop_reason": ["x"]}, + {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}] + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + for body in odd: + with self.subTest(body=body): + raw = json.dumps(body).encode() + Upstream.next = (200, "application/json", raw) + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)), raw) + self.assertEqual({k: v for k, v in r.calls("u0")[-1]["reply"].items() if k in NEW}, {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"[1, 2]", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) + + def test_what_the_record_read_before_is_read_as_before(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage stay as the relay read + them before (it took nothing from a ``response.failed`` event), its error beside them.""" + body = sse({"type": "response.output_text.delta", "delta": "part"}, + {"type": "response.failed", "response": {"object": "response", "status": "failed", + "error": {"code": "server_error", "message": "failed"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], + "usage": {"input_tokens": 5, "output_tokens": 2}}}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "", "tool_calls": [], "usage": None}) + self.assertEqual(relay.reply_facts("/responses", body, True), + {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) + + def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): + """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own + is added, and a unit's failure reads it as before (a 200 is no failure).""" r = self.relay(model="m-run") errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", {"code": 502, "message": "Provider returned error"}), - (sse({"id": "c1", "provider": "Prov A", "error": {"code": "server_error", "message": "upstream ended"}, - "choices": [{"delta": {}, "finish_reason": "error"}]}), "text/event-stream", + (sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, + "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", {"code": "server_error", "message": "upstream ended"}), (sse({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), "text/event-stream", {"type": "overloaded_error", "message": "Overloaded"})] for body, kind, said in errors: with self.subTest(said=said): Upstream.next = (200, kind, body) - post(r.url("u0") + "/chat/completions", body_for("/chat/completions", kind == "text/event-stream")) + streamed = kind == "text/event-stream" + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", streamed)), body) c = r.calls("u0")[-1] self.assertEqual((c["status"], c["reply"]["error"]), (200, said)) - self.assertIn(said["message"], c["error"]) - self.assertEqual(relay.failure(r.calls("u0")), ("ApiError", c["error"])) - Upstream.next = (200, "application/json", b'{"error":{"code":429,"message":"Rate limit exceeded"}}') - post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) - self.assertEqual(relay.failure(r.calls("u0"))[0], "RateLimitError") - post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) # answered: the unit recovered - self.assertIsNone(relay.failure(r.calls("u0"))) - - def test_a_pinned_chat_request_asks_for_the_routing_facts_and_reads_its_provider_there(self): - r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) - Upstream.next = (200, "application/json", json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, - openrouter_metadata={"attempts": [{"provider": "Prov X", "status": 503}, - {"provider": "Prov A", "status": 200}]})).encode()) - post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) - self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled") + self.assertNotIn("error", c) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", body, streamed)) + self.assertIsNone(relay.failure(r.calls("u0"))) + + def test_the_routing_facts_are_asked_of_openrouter_alone_and_always_as_enabled(self): + meta = json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, + openrouter_metadata={"attempts": [{"provider": "Prov X", "status": 503}, + {"provider": "Prov A", "status": 200}]})).encode() + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + Upstream.next = (200, "application/json", meta) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False), {"X-OpenRouter-Metadata": "disabled"}) + got = [v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"] + self.assertEqual(got, ["enabled"]) self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") - post(r.url("u0") + "/messages", body_for("/messages", False)) # its reply names its provider: nothing asked - self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) - plain = self.relay(model="m-run") - post(plain.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0") + "/messages", body_for("/messages", False)) # not a Chat request: nothing asked self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + for kw in ({"serving": {"m-run": ["prov-a"]}}, {"facts": True}): # another provider's address; no pin + with self.subTest(kw=kw): + other = self.relay(model="m-run", **kw) + post(other.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + self.assertTrue(relay.Relay("https://openrouter.ai/api/v1", "k", os.path.join(self.tmp, "or")).facts) + self.assertFalse(relay.Relay("https://api.openai.com/v1", "k", os.path.join(self.tmp, "oa")).facts) def test_without_a_pin_every_request_goes_on_byte_for_byte_as_before(self): """What a relay sent before pins existed: the caller's body with the model set (and a streamed Chat - Completions request's usage flag), serialised once; a pin on another model changes none of it.""" + Completions request's usage flag), serialised once, and the caller's headers but its credentials. The + three shapes, streamed or not, with and without a routing field of the caller's, as the run's model, a + side model and a judge's route's model; with no pin, or a pin on another model.""" for serving in (None, {"m-other": ["prov-a"]}): - r = self.relay(model="m-run", serving=serving) + r = self.relay(model="m-run", side_models=("m-side",), serving=serving) for (path, stream) in SHAPES: for extra in ({}, {"provider": {"order": ["prov-z"]}}): - with self.subTest(serving=serving, path=path, stream=stream, extra=extra): - asked = dict(body_for(path, stream), **extra) - post(r.url("u0") + path, asked) - expected = dict(asked, model="m-run") - if stream and path == "/chat/completions": - expected.setdefault("stream_options", {})["include_usage"] = True - self.assertEqual(Upstream.seen[-1][2], json.dumps(expected).encode()) - self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + for url, asked, model in ((r.url("u0"), "asked", "m-run"), (r.url("u0"), "m-side", "m-side"), + (r.url("u0-judge", model="m-judge"), "asked", "m-judge")): + with self.subTest(serving=serving, path=path, stream=stream, extra=extra, model=model): + req = dict(body_for(path, stream), model=asked, **extra) + post(url + path, req, {"X-Caller": "kept"}) + expected = dict(req, model=model) + if stream and path == "/chat/completions": + expected.setdefault("stream_options", {})["include_usage"] = True + sent_path, headers, raw = Upstream.seen[-1] + self.assertEqual(raw, json.dumps(expected).encode()) + self.assertEqual(sent_path, "/v1" + path) + self.assertEqual({k: v for k, v in headers.items() if k not in ("host", "content-length")}, + {"content-type": "application/json", "x-caller": "kept", "accept": "*/*", + "accept-encoding": "identity", "authorization": "Bearer or-test", + "user-agent": headers["user-agent"]}) class TheSidecar(WithUpstream): @@ -251,6 +348,19 @@ def docker(args, **kw): (started,) = [a for a in ran if a[:1] == ["run"]] self.assertIn("m-run=prov-a", started[started.index("--served-by") + 1:]) + def test_the_relays_own_command_refuses_a_pin_it_cannot_take(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + for given in (["m=bad slug"], ["m=prov-a,prov-a"], ["m="], ["m=prov-a", "m=prov-b"], ["prov-a"]): + with self.subTest(given=given): + out = io.StringIO() + with redirect_stdout(out): + code = relay.serve(["--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, + *[w for g in given for w in ("--served-by", g)]]) + self.assertEqual(code, 2) + self.assertIn("error", json.loads(out.getvalue())) + def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): key = os.path.join(self.tmp, "key") with open(key, "w") as f: @@ -260,12 +370,18 @@ def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): "--served-by", "m-run=prov-a,prov-b/fp8"], cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) try: + ready, _, _ = select.select([p.stdout], [], [], 20) # its one line, or no more than 20 s + self.assertTrue(ready, "the relay did not say it listens within 20 s") url = json.loads(p.stdout.readline())["listening"] post(url + "/u0/v1/chat/completions", body_for("/chat/completions", False)) self.assertEqual(self.sent()["provider"], PIN) finally: p.terminate() - p.wait(10) + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + p.wait(10) p.stdout.close() @@ -281,41 +397,59 @@ def tearDown(self): install.HOME = self._home super().tearDown() + def make(self, **kw): + kw.setdefault("other", self.a) + return run.Run(kw.pop("bench", self.b), kw.pop("other"), model=kw.pop("model", "m-1"), provider=kw.pop("provider", "openrouter"), + env=kw.pop("env", {"OPENROUTER_API_KEY": "or-test"}), unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, **kw) + def test_the_option_reads_model_and_slugs_and_refuses_what_it_cannot_take(self): self.assertEqual(run.served_by(["prov-a", "m-judge=prov-b/fp8, prov-c"], "m-run"), {"m-run": ["prov-a"], "m-judge": ["prov-b/fp8", "prov-c"]}) for given, said in ((["m-run=prov-a,"], "not a serving provider's slug"), (["m-run=a b"], "not a serving provider's slug"), - (["prov-a,prov-a"], "twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), - (["prov-a"], "MODEL=SLUG")): + (["prov-a,prov-a"], "named twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), + (["m-run="], "not a serving provider's slug")): with self.subTest(given=given), self.assertRaisesRegex(run.Refused, said): - run.served_by(given, "m-run" if given != ["prov-a"] else "") + run.served_by(given, "m-run") + with self.assertRaisesRegex(run.Refused, "MODEL=SLUG"): + run.served_by(["prov-a"], "") + + def test_a_run_refuses_a_pin_it_cannot_take_however_it_is_given(self): + for pin in ({"m-1": []}, {"m-1": ["bad slug"]}, {"m-1": ["prov-a", "prov-a"]}, {"": ["prov-a"]}, {"m-1": "prov-a"}): + with self.subTest(pin=pin), self.assertRaisesRegex(run.Refused, "--served-by"): + self.make(served_by=pin) def test_a_provider_that_routes_no_model_to_several_is_refused_before_any_unit(self): for provider, env in (("openai", {"OPENAI_API_KEY": "sk-x"}), ("anthropic", {"ANTHROPIC_API_KEY": "x"}), ("custom", {"MODEL_API_URL": self.base})): with self.subTest(provider=provider), self.assertRaisesRegex(run.Refused, "no serving providers to choose among"): - run.Run(self.b, self.a, model="m-1", provider=provider, env=env, unsandboxed=True, - jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + self.make(provider=provider, env=env, served_by={"m-1": ["prov-a"]}) def test_a_model_the_run_does_not_call_is_refused_before_any_unit(self): with self.assertRaisesRegex(run.Refused, "names m-other, which this run does not call: it calls m-1"): - run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, - jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + self.make(served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + nop = definitions.control(dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), "nop") + with self.assertRaisesRegex(run.Refused, "names m-1, which this run does not call: it calls no model"): + self.make(other=nop, bench=dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), + served_by={"m-1": ["prov-a"]}) - def test_the_command_line_refuses_with_exit_2(self): + def test_the_command_line_refuses_each_with_exit_2(self): for patch in (mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_other", return_value=self.a), - mock.patch.object(cli, "_exit_on_term"), - mock.patch.object(cli.settings, "load"), mock.patch.object(progress, "ECHO", None)): + mock.patch.object(cli, "_exit_on_term"), mock.patch.object(cli.settings, "load"), + mock.patch.object(progress, "ECHO", None)): patch.start() self.addCleanup(patch.stop) for extra, said in ((["--provider", "openai", "--served-by", "prov-a"], "no serving providers"), - (["--provider", "openrouter", "--served-by", "m-2=prov-a"], "does not call"), - (["--provider", "openrouter", "--served-by", "m-1=bad slug"], "not a serving provider's slug")): + (["--served-by", "m-2=prov-a"], "does not call"), + (["--served-by", "m-1=bad slug"], "not a serving provider's slug"), + (["--served-by", "m-1="], "not a serving provider's slug"), + (["--served-by", "prov-a,prov-a"], "named twice"), + (["--served-by", "prov-a", "--served-by", "m-1=prov-b"], "m-1 twice")): with self.subTest(extra=extra), mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-x", "OPENROUTER_API_KEY": "or-test"}): out = io.StringIO() with redirect_stdout(out): - code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--jobs", - os.path.join(self.tmp, "jobs"), *extra]) + code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--provider", "openrouter", + "--jobs", os.path.join(self.tmp, "jobs"), *extra]) self.assertEqual(code, 2) self.assertTrue(out.getvalue().strip().splitlines()[-1].startswith("refused:"), out.getvalue()) self.assertIn(said, out.getvalue()) @@ -330,10 +464,12 @@ def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): self.assertTrue(all(json.loads(raw).get("provider") == {"order": ["prov-a"], "allow_fallbacks": False} for _, _, raw in Upstream.seen)) with open(os.path.join(job_dir, "config.json")) as f: - cfg = json.load(f) - self.assertEqual(cfg["standard"]["served_by"], {"m-1": ["prov-a"]}) - trials = {d.split("__")[0]: json.load(open(os.path.join(job_dir, d, "result.json")))["standard"] - for d in os.listdir(job_dir) if "__u" in d} + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + trials = {} + for d in os.listdir(job_dir): + if "__u" in d: + with open(os.path.join(job_dir, d, "result.json")) as f: + trials[d.split("__")[0]] = json.load(f)["standard"] n = len(Upstream.seen) self.assertEqual(trials["model"]["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) self.assertEqual(trials["good"]["served"], {"calls": {}, "unnamed": 0, "held": {"m-1": None}}) # it called no model @@ -343,6 +479,67 @@ def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): with open(os.path.join(job_dir, "result.json")) as f: self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + def test_a_run_without_the_option_sends_records_and_ends_as_before(self): + """Every request as before, the record's earlier fields as they were read, and the same verdicts, even + when the provider answers a 200 that holds an error: the relay reads that error beside the reply and the + unit ends as it did.""" + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + reply = dict({k: v for k, v in CHAT.items() if k != "provider"}, error={"code": 502, "message": "Provider returned error"}) + Upstream.always = (200, "application/json", json.dumps(reply).encode()) + job_dir = run.run(b, self.a, model="m-1", provider="custom", env={"MODEL_API_URL": self.base, "MODEL_API_KEY": "k"}, + unsandboxed=True, jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding") + self.assertTrue(Upstream.seen) + for _, headers, raw in Upstream.seen: + self.assertNotIn("provider", json.loads(raw)) + self.assertNotIn("x-openrouter-metadata", headers) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("served_by", json.load(f)["standard"]) + with open(os.path.join(job_dir, "model__u0002", "result.json")) as f: + res = json.load(f) + self.assertNotIn("exception_info", res) # a 200 is no failure: the unit's verdict stands + self.assertEqual(res["verifier_result"]["rewards"]["reward"], 0.0) + self.assertNotIn("served", res["standard"]) # no call answered, no pin: nothing to say + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + for c in calls: + self.assertEqual((c["status"], "error" in c), (200, False)) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", json.dumps(reply).encode(), False)) + self.assertEqual(c["reply"]["error"], {"code": 502, "message": "Provider returned error"}) + self.assertNotIn("routing", c["request"]) + + def test_the_summary_counts_every_route_of_the_unit(self): + """A judge on the pinned model, served by another provider than the agent's: the pin does not hold.""" + r = self.make(judge_model="m-1", served_by={"m-1": ["prov-a"]}) + call = lambda who: {"model": "m-1", "status": 200, "reply": {"served_by": who}} + routes = {"u0000": [call("Prov A")], "u0000-judge": [call("Prov Z")], "u0000-simulator": [call("Prov A")]} + rr = mock.Mock(mcp_listeners=[], unit_image="") + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 1} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit", "note_refusal"): + stack.enter_context(mock.patch.object(r, name)) + stack.enter_context(mock.patch.object(r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = r.unit(0, "good") + self.assertEqual(res["standard"]["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + self.assertEqual(res["agent_result"]["n_model_calls"], 1) # the tokens and cost: the agent's calls, as before + + def test_the_benchmarks_own_relay_counts_in_the_job(self): + r = self.make(served_by={"m-1": ["prov-a"]}) + folder = os.path.join(r.job_dir, ".work", "relay-benchmark") + os.makedirs(folder) + with open(os.path.join(folder, "embed-1.jsonl"), "w") as f: + f.write(json.dumps({"model": "m-1", "status": 200, "reply": {"served_by": "Prov Z"}}) + "\n") + bench = job.served(r.benchmark_calls(), r.serving) + job.write_result(r.job_dir, r.name, 0, 1, [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 0, "held": {"m-1": True}}}}], + 1, served=bench) + with open(os.path.join(r.job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + def test_the_containers_unit_is_given_the_runs_pin(self): with mock.patch.object(container, "unusable", return_value=None): r = run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, backend="container", @@ -355,20 +552,33 @@ def test_the_containers_unit_is_given_the_runs_pin(self): class TheRecord(unittest.TestCase): + def call(self, model, who=None, status=200, error=None): + reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} + return {"model": model, "status": status, "reply": reply} + def test_who_served_a_units_calls_and_whether_the_pin_held(self): - def call(model, who=None, status=200, error=None): - reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} - return {"model": model, "status": status, "reply": reply} pin = {"m-1": ["google-ai-studio", "prov-b/fp8"], "m-judge": ["prov-c"]} - self.assertEqual(job.served([call("m-1", "Google AI Studio"), call("m-1", "Prov B"), call("m-1", status=429)], pin), + self.assertEqual(job.served([self.call("m-1", "Google AI Studio"), self.call("m-1", "Prov B"), self.call("m-1", status=429)], pin), {"calls": {"Google AI Studio": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": True, "m-judge": None}}) - self.assertEqual(job.served([call("m-1", "Prov Z"), call("m-1")], pin)["held"]["m-1"], "unknown") - self.assertEqual(job.served([call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered - self.assertEqual(job.served([call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) - self.assertIsNone(job.served([call("m-1")], None)) + self.assertEqual(job.served([self.call("m-1", "Prov Z"), self.call("m-1")], pin)["held"]["m-1"], "unknown") + self.assertEqual(job.served([self.call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered + self.assertEqual(job.served([self.call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) + self.assertIsNone(job.served([self.call("m-1", status=429)], None)) + + def test_a_name_reads_as_a_slug_by_its_case_and_spaces_alone(self): + pin = {"m-1": ["prov-a"]} + for name, held in (("Prov A", True), ("prov-a", True), ("PROV A", True), ("Prov/A", "unknown"), ("Prov_A", "unknown"), + ("Prov.A", "unknown"), ("Prov Ab", "unknown")): + with self.subTest(name=name): + self.assertEqual(job.served([self.call("m-1", name)], pin)["held"]["m-1"], held) + + def test_answered_calls_that_name_no_provider_are_counted(self): + known, unknown = job.served([self.call("m-1", "Prov A")]), job.served([self.call("m-1")]) + self.assertEqual(unknown, {"calls": {}, "unnamed": 1}) + self.assertEqual(job.served_total([{"standard": {"served": known}}, {"standard": {"served": unknown}}, {"standard": {}}]), + {"calls": {"Prov A": 1}, "unnamed": 1}) parts = [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 1, "held": {"m-1": True}}}}, - {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}, - {"standard": {}}] + {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}] self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) @@ -383,19 +593,48 @@ def tearDown(self): install.HOME = self._home shutil.rmtree(self.tmp, ignore_errors=True) - def test_shards_of_different_pins_are_refused(self): + def shards(self, *pins): jobs = [run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=os.path.join(self.tmp, "jobs"), - env={}, say=lambda s: None, shard=(i, 2), launch="L", name=f"run-shard-{i}of2") for i in (1, 2)] - for job_dir, pin in zip(jobs, ({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]})): + env={}, say=lambda s: None, shard=(i, len(pins)), launch="L", name=f"run-shard-{i}of{len(pins)}") + for i in range(1, len(pins) + 1)] + for job_dir, pin in zip(jobs, pins): path = os.path.join(job_dir, "config.json") with open(path) as f: cfg = json.load(f) cfg["standard"]["served_by"] = pin with open(path, "w") as f: json.dump(cfg, f) + return jobs + + def test_shards_of_different_pins_are_refused(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) with self.assertRaisesRegex(merge.Refused, "serving providers"): merge.check(jobs) self.assertEqual(merge.check(jobs, force=True)["differ"][0].split(":")[0], "serving providers") + jobs = self.shards({"m-1": ["prov-a", "prov-b"]}, {"m-1": ["prov-b", "prov-a"]}) # the order is the pin's + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + + def test_equal_pins_merge_whatever_the_order_of_their_models(self): + jobs = self.shards({"a": ["prov-a"], "b": ["prov-b"]}, {"b": ["prov-b"], "a": ["prov-a"]}) + self.assertEqual(merge.check(jobs)["differ"], []) + + def test_a_forced_merge_of_different_pins_holds_no_pin(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + for job_dir, who in zip(jobs, ("Prov A", "Prov B")): + trial = sorted(d for d in os.listdir(job_dir) if "__u" in d)[0] + path = os.path.join(job_dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["served"] = {"calls": {who: 1}, "unnamed": 0, "held": {"m-1": True}} + with open(path, "w") as f: + json.dump(res, f) + out = os.path.join(self.tmp, "merged") + merge.merge(jobs, out, force=True) + with open(os.path.join(out, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + with open(os.path.join(out, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) if __name__ == "__main__": From a0488669362d4df21455f72a87145dee40e4d3fd Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:28:39 +0100 Subject: [PATCH 05/24] docs(standard): say how the record reads who served a call, and what held cannot tell The pages now say that an error under a 200 is recorded and changes nothing else, that the record's copies are cleaned of credentials, that who served a run counts every route, how a name reads as a slug, and that held names a provider, never its endpoint variant. --- docs/cli/run.md | 11 ++++++----- docs/standard.md | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index a050165..8ee98e1 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -78,12 +78,13 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | --- | --- | | A slug | As the model's page on OpenRouter names its provider: `deepinfra`, `deepinfra/fp8`. Several are tried in their order, and no other provider answers | | Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | -| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A Chat Completions request also asks for OpenRouter's routing facts (`X-OpenRouter-Metadata: enabled`), where its reply names the provider that served it. A model with no pin is sent as before | -| Refused before any unit | A provider other than `openrouter`; a model the run does not call; a slug that is empty, malformed or given twice; a model named twice | +| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`), where its reply names the provider that served it. A model with no pin is sent as before | +| Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | | No provider can serve it | OpenRouter answers with an error and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, `reply.end` and `reply.error` | -| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug (lowercased, its words joined by dashes: `Google AI Studio` is `google-ai-studio`), `unknown` when one names none or another name, `null` with no answered call | -| Shards | `merge` refuses shards whose pins differ, unless `--force` | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`Google AI Studio` is `google-ai-studio`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | +| What `held` cannot tell | A reply names the provider, never its endpoint variant: `held` says that `deepinfra` answered, not that `deepinfra/fp8` did | +| Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | | The builder | A recipe has no field for it yet: a pinned run starts from the command line |
diff --git a/docs/standard.md b/docs/standard.md index 47341a8..802ee21 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -147,8 +147,8 @@ record); verifier/reward.txt and score.json | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | -| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`). A call with such an error is a failed call | -| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls by serving provider and, for each pinned model, `held` ([Run a benchmark](cli/run.md#serving-providers)) | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which names a provider and not its endpoint variant ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | ## Safety From 325e01469d388970c11c78412033a403083b1391 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:33:59 +0100 Subject: [PATCH 06/24] test(standard): run the relay's own command in a bounded process when it must refuse a pin --- app/tests/test_standard_served_by.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index 8a06141..7c1fb24 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -354,12 +354,13 @@ def test_the_relays_own_command_refuses_a_pin_it_cannot_take(self): f.write("or-test") for given in (["m=bad slug"], ["m=prov-a,prov-a"], ["m="], ["m=prov-a", "m=prov-b"], ["prov-a"]): with self.subTest(given=given): - out = io.StringIO() - with redirect_stdout(out): - code = relay.serve(["--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, - *[w for g in given for w in ("--served-by", g)]]) - self.assertEqual(code, 2) - self.assertIn("error", json.loads(out.getvalue())) + # a process of its own, bounded: a relay that took the pin would serve until stopped + got = subprocess.run([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", + "--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, + *[w for g in given for w in ("--served-by", g)]], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), capture_output=True, text=True, timeout=20) + self.assertEqual(got.returncode, 2, got.stdout) + self.assertIn("error", json.loads(got.stdout.splitlines()[0])) def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): key = os.path.join(self.tmp, "key") From fbcbcdfa40ec1eca6b4c0886d46e623781ac8052 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:40:34 +0100 Subject: [PATCH 07/24] docs(standard): say what held reads in a sentence without a contrast --- docs/cli/run.md | 2 +- docs/standard.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index 8ee98e1..891d2c5 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -83,7 +83,7 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | No provider can serve it | OpenRouter answers with an error and nothing else is tried; the call fails as any refused call does | | The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`Google AI Studio` is `google-ai-studio`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | -| What `held` cannot tell | A reply names the provider, never its endpoint variant: `held` says that `deepinfra` answered, not that `deepinfra/fp8` did | +| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra` | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | | The builder | A recipe has no field for it yet: a pinned run starts from the command line | diff --git a/docs/standard.md b/docs/standard.md index 802ee21..83c736d 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -148,7 +148,7 @@ record); verifier/reward.txt and score.json | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | | Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | -| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which names a provider and not its endpoint variant ([Run a benchmark](cli/run.md#serving-providers)) | +| Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | ## Safety From 8771cf01d407070b24c5f5664dcba578e50b7dd2 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:53:10 +0100 Subject: [PATCH 08/24] feat(builder): choose a model's serving providers on the Model step A run started from the builder could not pin its serving providers. On OpenRouter, the Model step now shows Served by under each model field (the model, the user simulator's and the judge's): the providers that may answer that model, tried in their order and no other, typed as slugs or added from the model's providers. Empty, the default, pins nothing. The recipe holds them as served_by, by field. The server checks them as --served-by is checked, refuses them for another provider or for a model no job calls, and compiles them into --served-by for each job that calls that model, by the name the run sends it as. Review shows them in its Model row. A saved recipe keeps them, and choosing another provider drops them. A recipe with no pin compiles to the command it did. The model's providers come from a new route, POST /new-run/endpoints, which asks OpenRouter's endpoint list as /new-run/models asks the model list: local and form-token guarded, the key kept on this machine and sent when set, the answer cached by the page for five minutes, and a fixed message when the list cannot be had, the field then taking a typed slug. --- app/nullius/providers.py | 48 ++++++ app/nullius/standard/recipe.py | 56 ++++++- app/static/builder/command-builder-core.js | 27 +++- app/static/builder/command-builder.css | 5 + app/static/builder/command-builder.js | 42 +++++- app/tests/test_standard_recipe_served_by.py | 154 ++++++++++++++++++++ app/ui_tests/test_served_by_builder.py | 91 ++++++++++++ app/web/builder.py | 18 +++ 8 files changed, 433 insertions(+), 8 deletions(-) create mode 100644 app/tests/test_standard_recipe_served_by.py create mode 100644 app/ui_tests/test_served_by_builder.py diff --git a/app/nullius/providers.py b/app/nullius/providers.py index 22c65bf..bf1d2da 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -16,6 +16,7 @@ import http.client import json import os +import re import secrets import socket import ssl @@ -211,6 +212,53 @@ def list_models(provider: str, env=None, *, opener=None) -> list[dict]: raise ValueError("Model list unavailable. Enter a model ID manually.") from None +# A model OpenRouter serves: its maker, a slash, its name (a variant after a colon: ``:free``). +_ROUTED = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._:-]*") +# A serving provider's slug, as ``--served-by`` takes it (``standard.relay.SLUG``). +_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def _endpoints(content: bytes) -> list[dict]: + """The serving providers of OpenRouter's endpoint list (``{"data": {"endpoints": [...]}}``), each + ``{slug, name, quantization}``: the slug its ``tag`` (a provider and its variant, ``deepinfra/fp8``), + else the provider's name lowercased with its spaces as dashes. ValueError when there is no list.""" + data = json.loads(content) + rows = ((data.get("data") or {}).get("endpoints") if isinstance(data, dict) and isinstance(data.get("data"), dict) else None) + if not isinstance(rows, list): + raise ValueError("missing endpoint list") + out = {} + for row in rows: + if not isinstance(row, dict): + continue + name = row.get("provider_name") if isinstance(row.get("provider_name"), str) else "" + tag = row.get("tag") if isinstance(row.get("tag"), str) else "" + slug = tag or re.sub(r"\s+", "-", name.strip().lower()) + if not slug or len(slug) > 128 or not _SLUG.fullmatch(slug) or slug in out: + continue + quant = row.get("quantization") if isinstance(row.get("quantization"), str) else "" + out[slug] = {"slug": slug, "name": (name or slug)[:128], "quantization": quant[:32]} + return sorted(out.values(), key=lambda e: (e["name"].casefold(), e["slug"])) + + +def list_endpoints(provider: str, model: str, env=None, *, opener=None) -> list[dict]: + """The serving providers OpenRouter lists for ``model`` (``GET /models/{author}/{slug}/endpoints``), for the + builder's Served by: asked as the model list is (its key when set, no redirect, nothing of the error echoed).""" + env = os.environ if env is None else env + provider, endpoint = split(provider) + if provider != "openrouter": + raise ValueError("Only OpenRouter routes a model to several providers.") + model = served_as(str(model or "").strip(), provider) + if not _ROUTED.fullmatch(model): + raise ValueError("Choose a model as OpenRouter names it, its maker first: maker/model.") + base, key = _where(provider, endpoint, env) + author, _, name = model.partition("/") + url = f"{base}/models/{urllib.parse.quote(author, safe='')}/{urllib.parse.quote(name, safe=':')}/endpoints" + try: + return _endpoints(_get(url, _bearer(key), opener)) + except (OSError, ValueError, TypeError, AttributeError, http.client.HTTPException): + raise ValueError("Provider list unavailable. Enter a provider's slug by hand.") from None + + # Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is # public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test. TESTED = ("openrouter", "openai", "anthropic", "custom") diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index e389fb7..53b602b 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -39,7 +39,7 @@ from nullius import hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove +from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay HARNESS = "standard" # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's. @@ -48,6 +48,8 @@ MAX_CONCURRENT = 32 BACKENDS = definitions.names.BACKENDS _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") +# A pin's field, as the Review's Model row names it (served_by, --served-by) +SERVED_LABEL = {"model": "model", "simulator": "user simulator", "judge": "judge"} _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$") @@ -280,7 +282,12 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it names have no price in the user's list (``nullius.prices``): a run - with a cap is refused without one, and the Limits step says so first.""" + with a cap is refused without one, and the Limits step says so first. + + ``served_by``: on OpenRouter, the serving providers each model field + pins, ``{model|simulator|judge: [slug, ...]}``, checked as ``--served-by`` + is and compiled into it for each job that calls that model, by the name + the run sends it as.""" d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) @@ -436,6 +443,20 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model + # the serving providers each model field pins, on OpenRouter (served_by: {model|simulator|judge: [slug, ...]}), + # checked as --served-by is (relay.serving_problems); an empty list pins nothing + roles = {"model": "the model", "simulator": "the user simulator's model", "judge": "the judge's model"} + pins = recipe.get("served_by") + served: Dict[str, List[str]] = {} + if pins not in (None, {}): + if not isinstance(pins, dict) or any(k not in roles for k in pins): + add("model", "Served by names the serving providers of the model, the user simulator's or the judge's.") + else: + served = {k: v for k, v in pins.items() if v not in (None, [], "")} + for problem in relay.serving_problems({roles[k]: v for k, v in served.items()}): + add("model", "Served by " + problem + ".") + if served and provider_ != "openrouter": + add("model", "Served by needs OpenRouter, the provider that routes a model to several: choose it, or leave Served by empty.") # the run's spending cap: an amount in USD, or none cap = recipe.get("max_usd") if cap in ("", None): @@ -489,6 +510,33 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, missing = unpriced([m for m in dict.fromkeys(called) if isinstance(m, str) and m]) if missing: add("limits", f"A spending cap needs a price for {', '.join(missing)}: update the price list in Settings, or set a price there.") + judge_def, sim_def = (b.get("score") or {}).get("judge"), b.get("simulator") + + def pinned(p: dict, used: Optional[set] = None) -> Dict[str, List[str]]: + """The --served-by of one job: each pinned role's model as the run sends it, where the job calls it + (ledger.models, as the run reads them); ``used`` collects the roles that apply.""" + sent = {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, + "judge": (judge_model or providers.served_as(judge_def["model"], provider)) if judge_def else None, + "simulator": (sim_model or providers.served_as(sim_def["model"], provider)) if sim_def else None} + called = ledger.models(p["o"], sent["model"], sent["judge"], sent["simulator"], None, control=p["control"]) + got: Dict[str, List[str]] = {} + for role, slugs in served.items(): + m = sent.get(role) + if not m or m not in called: + continue + if used is not None: + used.add(role) + if m in got and got[m] != list(slugs): + add("model", f"Served by pins {m} twice, with different providers: give its fields the same ones.") + got[m] = list(slugs) + return got + if served and not errors: + used: set = set() + for p in picks: + pinned(p, used) + for role in served: + if role not in used: + add("model", f"Served by: no job of this recipe calls {roles[role]}; leave it empty.") if errors: return out nouns = (b.get("units") or {}).get("nouns", "units") @@ -525,6 +573,7 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--max-usd-per-unit", f"{per_unit:.6f}".rstrip("0").rstrip(".")] if per_unit and spends(p) else []), *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), + *[w for m_, slugs in pinned(p).items() for w in ("--served-by", f"{m_}={','.join(slugs)}")], "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], *(["--backend", "container"] if backend == "container" else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), @@ -608,7 +657,8 @@ def row(p: dict) -> list: [STEP["model"], "Model", "none: a control calls no model" if out["keyless"] else f"{providers.label(providers.choice(provider, str(recipe.get('endpoint') or '')))} ยท " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only") + (f" ยท user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "") - + (f" ยท judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")], + + (f" ยท judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + + "".join(f" ยท {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" ยท {conc} at a time".replace(" ", " ") + (" of each dataset" if many and isinstance(limit, int) else "") diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index d20fd5b..85b1943 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -79,6 +79,24 @@ } return r; } + // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are + // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. + const SERVED_ROLES = ['model', 'simulator', 'judge']; + const SERVED_SLUG = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9._-]+)*$/; + function servedRoles(r) { + const b = standardBench(r); + if (!b || r.provider !== 'openrouter' || standardKeyless(r)) return []; + return SERVED_ROLES.filter(k => k === 'model' || (k === 'simulator' && b.simulator) || (k === 'judge' && b.judge)); + } + // A field's text, "deepinfra, together", as the pin's list; an empty field pins nothing. + const servedList = text => String(text || '').split(',').map(s => s.trim()).filter(Boolean); + function setServed(r, role, slugs) { + const list = Array.isArray(slugs) ? slugs : servedList(slugs); + r.served_by = {...(r.served_by || {})}; + if (list.length) r.served_by[role] = list; else delete r.served_by[role]; + if (!Object.keys(r.served_by).length) delete r.served_by; + return r; + } function createStandard(benchmark = '', other = '') { const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0]; const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [], @@ -141,6 +159,10 @@ // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); + // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) + if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) + for (const role of SERVED_ROLES) { const v = raw.served_by[role]; + if (Array.isArray(v) && v.length && v.every(s => typeof s === 'string' && SERVED_SLUG.test(s.trim()))) setServed(r, role, v.map(s => s.trim())); } r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; else r.model = bareModel(r.model); @@ -217,7 +239,8 @@ const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel); const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r); // The Model step's choice into the recipe: its provider, and for a named endpoint its id. - function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); } + // a provider that routes no model to several takes no Served by: a pin made for OpenRouter is dropped with it + function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); if (r.provider !== 'openrouter') delete r.served_by; } // Why Run cannot start this recipe here, or '': its provider is not one of those. A recipe naming one keeps it, // said, until another is chosen - never switched behind the person's back. The server refuses the same // (web/execute.py, provider_problem). @@ -759,5 +782,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index e37e016..290a3e9 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -246,3 +246,8 @@ #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +/* Served by: under its model field, set in from it: the typed list, then the providers to add from */ +#nullius-builder .std-served { margin: -10px 0 22px 18px; padding-left: 14px; border-left: 2px solid var(--nb-line); font-size: 14px; gap: 8px; } +#nullius-builder .std-served label { display: grid; gap: 6px; } +#nullius-builder .std-served select { max-width: 420px; } +@media(max-width:520px) { #nullius-builder .std-served { margin-left: 6px; padding-left: 10px; } } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 66f9cf8..74fba43 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -599,12 +599,13 @@ return `

Choose the model.

` + modelChoiceCards(rec) + providerField(rec, C.PROVIDERS) + field('Model', 'model', rec.model, {list:'nb-models', placeholder:paper && C.paperListed(rec) ? 'Choose a paper model' : 'Search or enter a model ID'}) + + servedField(rec, 'model', rec.model) // the benchmark's other models: the user simulator its conversations need, the judge that grades - each its official one // unless named; like the paper, the official one; like the leaderboard, its own and the options it fixes + (b.simulator ? (paper ? fixed('Model for the user simulator', lb && lb.simulator || b.simulator, lb && lb.simulator ? 'leaderboard' : 'official') : field('Model for the user simulator', 'simulator_model', rec.simulator_model || '', {list: 'nb-models', placeholder: b.simulator + ' (official)', - hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) : '') + hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) + servedField(rec, 'simulator', (!paper && rec.simulator_model) || lb && lb.simulator || b.simulator) : '') + (b.judge ? (paper ? fixed('Model for the judge', lb && lb.judge || b.judge, lb && lb.judge ? 'leaderboard' : 'official') : field('Model for the judge', 'judge_model', rec.judge_model || '', {list: 'nb-models', placeholder: b.judge + ' (official)', - hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) : '') + hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) + servedField(rec, 'judge', (!paper && rec.judge_model) || lb && lb.judge || b.judge) : '') + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, v, 'leaderboard')).join('') : '') + keyLine(rec); } @@ -643,6 +644,36 @@ const select = (label,key,value,options) => ``; const fixed = (label,value,whose='official') => ``; const note = (title,text) => `
${title}${text}
`; + // Served by: under a model field, on OpenRouter alone, the serving providers that may answer it, in order (recipe.py, + // served_by); typed as slugs, or added from the model's providers (POST /new-run/endpoints) + const SERVED_NAME = {model: 'model', simulator: 'user simulator', judge: 'judge'}; + const servedField = (rec, role, model) => !C.servedRoles(rec).includes(role) ? '' : + `
` + + `` + + `OpenRouter's providers for this model, tried in this order and no other. Empty: any of them.
`; + const servedCache = new Map(); + function servedList(model) { + let entry = servedCache.get(model); + if (!entry || Date.now() - entry.at > 300000) { + entry = {at: Date.now(), promise: nulliusFetch('/new-run/endpoints', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': schema.csrf}, body: JSON.stringify({provider: 'openrouter', model})}) + .then(x => x.json()).then(x => ({...x, endpoints: Array.isArray(x.endpoints) ? x.endpoints : []})) + .catch(() => ({ok: false, endpoints: [], message: "Provider list unavailable. Enter a provider's slug by hand."}))}; + servedCache.set(model, entry); + } + return entry.promise.then(x => { if (!x.ok) servedCache.delete(model); return x; }); + } + function fillServed() { + root.querySelectorAll('select[data-served-add]').forEach(async select => { + const model = select.dataset.servedModel; + if (!model) return; + const got = await servedList(model); + if (!select.isConnected || select.dataset.servedModel !== model) return; + const first = select.options[0]; + if (!got.ok || !got.endpoints.length) { first.textContent = got.ok ? 'No provider listed: type a slug' : got.message; return; } + first.textContent = 'Add a providerโ€ฆ'; + for (const e of got.endpoints) { const o = document.createElement('option'); o.value = e.slug; o.textContent = e.name + ' ยท ' + e.slug + (e.quantization ? ' ยท ' + e.quantization : ''); select.append(o); } + }); + } function customRows(agent = '') { // '' lists the shared entries (every agent, and the other owners); an id lists that agent's own --ak. @@ -728,7 +759,7 @@ const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec); html='

Review the experiment.

'+rows.map(([i,t,v])=>`
${t}${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'
':'')+runPanel(); } - $('#nb-form').innerHTML=html+'

';applyPick(); + $('#nb-form').innerHTML=html+'

';applyPick();fillServed(); if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();} $('#nb-back').hidden=step===S.benchmark; $('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' โ†’'; @@ -946,6 +977,7 @@ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} + if(d.servedBy!==undefined){C.setServed(rec,d.servedBy,el.value);refreshPlan();return;} if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; const value=el.type==='number'?(el.value===''?'':Number(el.value)):el.value; @@ -976,6 +1008,10 @@ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} + // a provider added to a Served by, after the ones already there; the field shows the list, in its order + else if(d.servedAdd!==undefined){const rec=r(),role=d.servedAdd,now=(rec.served_by||{})[role]||[];if(el.value&&!now.includes(el.value))C.setServed(rec,role,[...now,el.value]);el.value='';const f=root.querySelector(`[data-served-by="${role}"]`);if(f)f.value=((r().served_by||{})[role]||[]).join(', ');refreshPlan();} + // a model chosen: its Served by lists that model's providers + else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan();} else if(d.stdVersion!==undefined){C.setStandardBenchmark(r(),el.value);render();} // an option at its default is left out: the recipe names only what was changed else if(d.benchOptSelect!==undefined){const rec=r(),o=((C.standardBench(rec)||{}).options||{})[d.benchOptSelect]||{};rec.bench_options={...(rec.bench_options||{})};const v=el.value.trim();if(!v||v===o.default)delete rec.bench_options[d.benchOptSelect];else rec.bench_options[d.benchOptSelect]=v;refreshPlan();} diff --git a/app/tests/test_standard_recipe_served_by.py b/app/tests/test_standard_recipe_served_by.py new file mode 100644 index 0000000..d1a3a70 --- /dev/null +++ b/app/tests/test_standard_recipe_served_by.py @@ -0,0 +1,154 @@ +"""The builder's Served by: a recipe's ``served_by`` (the serving providers +each model field pins, on OpenRouter) is checked as ``--served-by`` is and +compiled into it for each job that calls that model, by the name the run +sends it as; a recipe with none compiles to the command it did. The model's +providers come from OpenRouter's endpoint list, asked as the model list is.""" +from __future__ import annotations + +import io +import json +import os +import unittest +import urllib.error + +from nullius import providers +from nullius.standard import definitions, recipe + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +NOP = {"nop": {"source": "nullius", "from": "nothing said"}} + + +def defs(bench=None): + d = {k: definitions.load(k, ROOT)[0] for k in ("benchmark", "agent", "capability")} + if bench is not None: + d["benchmark"] = [bench if x["key"] == bench["key"] else x for x in d["benchmark"]] + return d + + +def fakebench(**kw): + return dict(definitions.find(definitions.load("benchmark", ROOT)[0], "fakebench"), **kw) + + +def judged(): + b = fakebench(controls=NOP) + return dict(b, score=dict(b["score"], judge={"model": "gpt-4o-mini", "says": "a judge"})) + + +def a_recipe(**kw): + r = {"version": 1, "harness": "standard", "name": "fake-0101-120000", "title": "t", "benchmark": "fakebench", "dataset": "all", + "other": "fake", "provider": "openrouter", "model": "openai/gpt-5-mini", "limit": 2, "concurrency": 2, "unsandboxed": True, + "where": "local"} + r.update(kw) + return r + + +class TheRecipe(unittest.TestCase): + def test_a_pin_compiles_to_the_option_and_shows_in_the_review(self): + out = recipe.compile(a_recipe(served_by={"model": ["deepinfra", "together/fp8"]}), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--served-by openai/gpt-5-mini=deepinfra,together/fp8", out["script"]) + model_row = next(row[2] for row in out["review"] if row[1] == "Model") + self.assertIn("model served by deepinfra, together/fp8", model_row) + + def test_each_job_is_pinned_where_it_calls_the_model(self): + """A control beside the agent calls no model of its own: its job takes no pin for the run's model; a judge + pin goes on every job of a judged benchmark, the control's included, by the name OpenRouter serves it as.""" + d = defs(judged()) + out = recipe.compile(a_recipe(others=["fake", "nop"], served_by={"model": ["deepinfra"], "judge": ["prov-j"]}), d) + self.assertEqual(out["errors"], []) + scripts = [p["script"] for p in out["parts"]] + self.assertIn("--served-by openai/gpt-5-mini=deepinfra", scripts[0]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[0]) + self.assertNotIn("openai/gpt-5-mini=deepinfra", scripts[1]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[1]) + row = next(r for r in out["review"] if r[1] == "Model")[2] + self.assertIn("judge served by prov-j", row) + + def test_a_pin_is_refused_as_the_option_is(self): + for kw, said in (({"provider": "openai", "model": "gpt-5-mini", "served_by": {"model": ["deepinfra"]}}, "needs OpenRouter"), + ({"served_by": {"model": ["bad slug"]}}, "not a serving provider's slug"), + ({"served_by": {"model": ["deepinfra", "deepinfra"]}}, "named twice"), + ({"served_by": {"agent": ["deepinfra"]}}, "Served by names"), + ({"served_by": ["deepinfra"]}, "Served by names"), + ({"served_by": {"judge": ["prov-j"]}}, "no job of this recipe calls the judge's model"), + ({"served_by": {"model": "deepinfra"}}, "Served by")): + with self.subTest(kw=kw): + out = recipe.compile(a_recipe(**kw), defs()) + self.assertTrue(out["errors"], out["errors"]) + self.assertTrue(all(e["step"] == recipe.STEP["model"] for e in out["errors"]), out["errors"]) + self.assertTrue(any(said in e["message"] for e in out["errors"]), out["errors"]) + self.assertEqual(out["script"], "") + + def test_one_model_in_two_fields_takes_one_list(self): + d = defs(judged()) + same = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["a"]}), d) + self.assertEqual(same["errors"], []) + lines = [line for line in same["script"].splitlines() if "--served-by" in line] + self.assertTrue(lines and all(line.count("--served-by") == 1 for line in lines)) # each way of starting it: once + other = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["b"]}), d) + self.assertIn("pins openai/gpt-4o-mini twice", other["errors"][0]["message"]) + + def test_a_recipe_with_no_pin_compiles_to_the_command_it_did(self): + plain = recipe.compile(a_recipe(), defs()) + self.assertEqual(plain["errors"], []) + self.assertNotIn("--served-by", plain["script"]) + for none in ({}, {"model": []}, None): + with self.subTest(none=none): + self.assertEqual(recipe.compile(a_recipe(served_by=none), defs())["script"], plain["script"]) + openai = recipe.compile(a_recipe(provider="openai", model="gpt-5-mini", served_by={}), defs()) + self.assertEqual(openai["errors"], []) + + +class TheProviders(unittest.TestCase): + """``providers.list_endpoints``: OpenRouter's endpoint list, a fake opener in the place of the network.""" + + def opener(self, body=None, error=None): + seen = [] + + def opener(req, timeout=None): + seen.append(req) + if error: + raise error + return io.BytesIO(json.dumps(body).encode()) + return opener, seen + + def test_a_models_providers_by_slug_and_name(self): + body = {"data": {"id": "meta-llama/llama-3.3-70b-instruct", "endpoints": [ + {"provider_name": "DeepInfra", "tag": "deepinfra/fp8", "quantization": "fp8"}, + {"provider_name": "Together", "tag": "together"}, + {"provider_name": "Google AI Studio"}, # no tag: its name as a slug reads + {"provider_name": "Bad", "tag": "bad slug"}, "not a row", {"tag": 7}]}} + opener, seen = self.opener(body) + got = providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + self.assertEqual(got, [{"slug": "deepinfra/fp8", "name": "DeepInfra", "quantization": "fp8"}, + {"slug": "google-ai-studio", "name": "Google AI Studio", "quantization": ""}, + {"slug": "together", "name": "Together", "quantization": ""}]) + (req,) = seen + self.assertEqual(req.full_url, "https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints") + self.assertIsNone(req.get_header("Authorization")) # no key set: the list is asked without one + self.assertEqual(req.get_method(), "GET") + + def test_the_key_goes_as_for_the_model_list_and_a_bare_name_gets_its_maker(self): + opener, seen = self.opener({"data": {"endpoints": []}}) + providers.list_endpoints("openrouter", "gpt-4o-mini", env={"OPENROUTER_API_KEY": "or-test"}, opener=opener) + self.assertEqual(seen[0].full_url, "https://openrouter.ai/api/v1/models/openai/gpt-4o-mini/endpoints") + self.assertEqual(seen[0].get_header("Authorization"), "Bearer or-test") + + def test_what_cannot_be_asked_or_read_is_said_in_fixed_words(self): + for kw in ({"error": urllib.error.URLError("refused at https://openrouter.ai?key=x")}, {"body": {"error": "no"}}, + {"body": {"data": {"endpoints": "none"}}}): + with self.subTest(kw=kw): + opener, _ = self.opener(**kw) + with self.assertRaisesRegex(ValueError, r"^Provider list unavailable\. Enter a provider's slug by hand\.$"): + providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + opener, seen = self.opener({}) + with self.assertRaisesRegex(ValueError, "Only OpenRouter"): + providers.list_endpoints("openai", "gpt-4o", env={"OPENAI_API_KEY": "x"}, opener=opener) + for model in ("no-maker", "a/b/../../c", "maker/model?x=1", ""): + with self.subTest(model=model), self.assertRaisesRegex(ValueError, "maker/model"): + providers.list_endpoints("openrouter", model, env={}, opener=opener) + self.assertEqual(seen, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_served_by_builder.py b/app/ui_tests/test_served_by_builder.py new file mode 100644 index 0000000..52ac3c3 --- /dev/null +++ b/app/ui_tests/test_served_by_builder.py @@ -0,0 +1,91 @@ +"""The Model step's Served by: the route that lists a model's providers on +OpenRouter (local, form-token guarded, asked as the model list is), and the +recipe's ``served_by`` in the builder's core script (its fields on +OpenRouter alone, kept by a saved recipe, dropped with another provider).""" +import json +import os +import unittest +from unittest.mock import patch + +from .builder_support import CompilerCase +from .test_routes import web + +LIST = [{'slug': 'deepinfra/fp8', 'name': 'DeepInfra', 'quantization': 'fp8'}, {'slug': 'together', 'name': 'Together', 'quantization': ''}] + + +class ProviderList(unittest.TestCase): + def setUp(self): + self.client = web.app.test_client() + with self.client.session_transaction() as session: + session['csrf'] = 'served-test-token' + self.headers = {'X-CSRF-Token': 'served-test-token'} + + def ask(self, body, **kw): + return self.client.post('/new-run/endpoints', json=body, headers=kw.pop('headers', self.headers), **kw) + + def test_a_models_providers_on_openrouter(self): + with patch('web.builder.providers.list_endpoints', return_value=LIST) as fetch: + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {'ok': True, 'endpoints': LIST}) + fetch.assert_called_once_with('openrouter', 'meta-llama/llama-3.3-70b-instruct') + + def test_a_list_that_cannot_be_had_is_no_empty_success(self): + with patch('web.builder.providers.list_endpoints', side_effect=ValueError("Provider list unavailable. Enter a provider's slug by hand.")): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 502) + self.assertEqual(response.json, {'ok': False, 'endpoints': [], 'message': "Provider list unavailable. Enter a provider's slug by hand."}) + + def test_no_key_asks_without_one(self): + """OpenRouter's endpoint list is public: with no key set, it is asked with no Authorization header.""" + sent = [] + + def get(url, headers, opener=None): + sent.append((url, headers)) + return json.dumps({'data': {'endpoints': [{'provider_name': 'Together', 'tag': 'together'}]}}).encode() + env = {k: v for k, v in os.environ.items() if k != 'OPENROUTER_API_KEY'} + with patch.dict(os.environ, env, clear=True), patch('web.builder.providers._get', side_effect=get): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.json['endpoints'], [{'slug': 'together', 'name': 'Together', 'quantization': ''}]) + self.assertEqual(sent, [('https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints', {})]) + + def test_guards_and_another_provider_do_not_fetch(self): + with patch('web.builder.providers.list_endpoints') as fetch: + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, headers={}).status_code, 403) + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, environ_overrides={'REMOTE_ADDR': '192.0.2.4'}).status_code, 403) + for body in ({'provider': 'openai', 'model': 'gpt-4o'}, {'provider': 'openrouter'}, {'provider': 'openrouter', 'model': ' '}, []): + with self.subTest(body=body): + self.assertEqual(self.ask(body).status_code, 400) + fetch.assert_not_called() + + +class ServedInTheCore(CompilerCase): + """command-builder-core.js: which model fields take a Served by, and the recipe's ``served_by``.""" + + def test_the_fields_show_on_openrouter_alone(self): + out = self.js('const b=C.standard().benchmarks.find(x=>x.simulator&&x.judge)||C.standard().benchmarks.find(x=>x.judge);' + 'const r=C.createStandard(b.key);r.model_choice="any";const got={};' + 'for(const p of ["openrouter","openai","custom"]){C.setChoice(r,p);got[p]=C.servedRoles(r);}' + 'process.stdout.write(JSON.stringify([got,!!b.simulator]));') + roles, simulated = out + self.assertEqual(roles['openrouter'], ['model'] + (['simulator'] if simulated else []) + ['judge']) + self.assertEqual((roles['openai'], roles['custom']), ([], [])) + + def test_a_saved_recipe_keeps_its_pins_and_another_provider_drops_them(self): + raw = {'version': 1, 'harness': 'standard', 'benchmark': '', 'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct', + 'served_by': {'model': ['deepinfra/fp8', ' together '], 'judge': ['bad slug'], 'agent': ['x'], 'simulator': []}} + out = self.js('const b=C.standard().benchmarks[0];const raw=' + json.dumps(raw) + ';raw.benchmark=b.key;' + 'const r=C.fromJSON(JSON.stringify(raw));const kept=JSON.parse(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model","a, b ,");const typed=r.served_by.model;C.setServed(r,"model","");const emptied=r.served_by===undefined;' + 'C.setServed(r,"model",["a"]);C.setChoice(r,"openai");' + 'process.stdout.write(JSON.stringify([kept,typed,emptied,r.served_by===undefined]));') + self.assertEqual(out, [{'model': ['deepinfra/fp8', 'together']}, ['a', 'b'], True, True]) + + def test_a_recipe_with_no_pin_is_saved_as_before(self): + out = self.js('const b=C.standard().benchmarks[0];const r=C.createStandard(b.key);C.setChoice(r,"openrouter");' + 'const again=C.fromJSON(JSON.stringify(r));process.stdout.write(JSON.stringify(["served_by" in r,"served_by" in again]));') + self.assertEqual(out, [False, False]) + + +if __name__ == '__main__': + unittest.main() diff --git a/app/web/builder.py b/app/web/builder.py index a649b4c..e069744 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -390,6 +390,24 @@ def new_run_registry(): return _REGISTRY_SCAN +@app.route("/new-run/endpoints", methods=["POST"]) +def new_run_endpoints(): + """The serving providers OpenRouter lists for one model, for the Model step's Served by; asked as the + model list is, with the key kept on this machine.""" + shared._require_csrf() + if not shared_local_client(): + abort(403) + body = request.get_json(silent=True) + provider = body.get("provider") if isinstance(body, dict) else None + model = body.get("model") if isinstance(body, dict) else None + if providers.split(provider)[0] != "openrouter" or not isinstance(model, str) or not model.strip(): + return jsonify(ok=False, endpoints=[], message="Only OpenRouter routes a model to several providers: choose it and a model."), 400 + try: + return jsonify(ok=True, endpoints=providers.list_endpoints(provider, model)) + except ValueError as error: + return jsonify(ok=False, endpoints=[], message=str(error)), 502 + + @app.route("/new-run/models", methods=["POST"]) def new_run_models(): """Return the selected provider's catalogue, keeping its key on this machine.""" From c0ea2bb3ef8e5a9cbf0afc6839c4bf58d21c3c69 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 15:53:10 +0100 Subject: [PATCH 09/24] docs(builder): describe Served by on the Model step and its list of a model's providers --- docs/builder.md | 2 +- docs/cli/run.md | 2 +- docs/settings/model-providers.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/builder.md b/docs/builder.md index 170ac16..2528907 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -26,7 +26,7 @@ Then [read its results](results.md). | Runner | **Nullius**, [the standard](standard.md#from-the-app), or **Harbor** with its pin ([Which Harbor](#which-harbor)). A choice only when the benchmark runs on both; Nullius is preselected | | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A missing or expired credential is named once, linked to **Settings โ†’ Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent ยท Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | -| Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled | +| Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, under each model field when the provider is OpenRouter: the serving providers that may answer that model, typed as slugs or added from the model's providers, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit | | Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | diff --git a/docs/cli/run.md b/docs/cli/run.md index 891d2c5..def6026 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -85,7 +85,7 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`Google AI Studio` is `google-ai-studio`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | | What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra` | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | -| The builder | A recipe has no field for it yet: a pinned run starts from the command line | +| The builder | *Model* step, **Served by** under each model field, on OpenRouter alone: the recipe's `served_by`, by field (`model`, `simulator`, `judge`), compiled into `--served-by` for each job that calls that model. Its list of a model's providers comes from OpenRouter (`/models/{maker}/{model}/endpoints`), asked as the model list is; when it cannot be had, a slug typed by hand is taken |
diff --git a/docs/settings/model-providers.md b/docs/settings/model-providers.md index 5c8362b..b986737 100644 --- a/docs/settings/model-providers.md +++ b/docs/settings/model-providers.md @@ -6,7 +6,7 @@ The key or endpoint a run's model calls go to. | --- | --- | --- | | TypeSafe AI | `TYPESAFE_API_KEY` | System One judgement only | | OpenAI | `OPENAI_API_KEY` | api.openai.com | -| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed; a run may choose the serving providers of a model ([`--served-by`](../cli/run.md#serving-providers)) | +| OpenRouter | `OPENROUTER_API_KEY` | `https://openrouter.ai/api/v1`, fixed; a run may choose the serving providers of a model: **Served by** on the builder's Model step, or [`--served-by`](../cli/run.md#serving-providers) | | Anthropic | `ANTHROPIC_API_KEY` | Standard runs only, through the relay | | OpenAI-compatible | A name, a URL, an optional key, per endpoint | Ollama, vLLM, LM Studio or a hosted service, API path included | From c47d9ffcab2a9defef0eef069a8a74fe38eef214 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:06:44 +0100 Subject: [PATCH 10/24] fix(builder): set Served by below its model field's row, clear of its line --- app/static/builder/command-builder.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 290a3e9..af45dec 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -247,7 +247,7 @@ #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } /* Served by: under its model field, set in from it: the typed list, then the providers to add from */ -#nullius-builder .std-served { margin: -10px 0 22px 18px; padding-left: 14px; border-left: 2px solid var(--nb-line); font-size: 14px; gap: 8px; } +#nullius-builder .std-served { margin: 10px 0 22px 18px; padding: 2px 0 2px 14px; border-left: 2px solid var(--nb-line); font-size: 14px; gap: 8px; } #nullius-builder .std-served label { display: grid; gap: 6px; } #nullius-builder .std-served select { max-width: 420px; } @media(max-width:520px) { #nullius-builder .std-served { margin-left: 6px; padding-left: 10px; } } From 5c57a3828a8f8e4cd650c2abcb30e3c30a4f4c6b Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:40:55 +0100 Subject: [PATCH 11/24] fix(standard): hold a pin to the provider that answered and to the model it was set for A review of the pin found records and summaries that could read held where another provider answered, credentials that could reach a record, and pins that the builder dropped or moved to another model. Now: - The provider recorded is the one of the reply's end: a stream's latest event that names one, and the last attempt of OpenRouter's routing facts; an earlier, failed attempt is never taken, and a last attempt that names none gives none. - An error keeps its code, type and message alone, of those kinds; the rest of the object is not kept. A recorded routing object's keys are cleaned as its values are. - A merged job's summary is made of each shard's whole one, its benchmark relay's calls and an unknown included. - A recipe's pins are resolved by model, then given to every job that calls that model, a control's judge job included. The compile route returns each field's model as the run sends it, and Served by lists that model's providers. - A control is refused a pin on its simulated user's model, which it never calls. relay serve refuses a pin unless its base is OpenRouter's (or --routing is given) and the model is one it sends; a container's relay is given the pins of the models it sends. - The builder clears a field's pin when its model changes, a preset or a benchmark included; keeps a saved pin as written, for the server to refuse what is wrong in sight; keeps an empty part of a typed list; puts its help in a tooltip; and keeps the keyboard on the model field once the form is drawn again. --- app/nullius/standard/container.py | 5 +- app/nullius/standard/job.py | 5 +- app/nullius/standard/merge.py | 16 +++-- app/nullius/standard/recipe.py | 46 +++++++----- app/nullius/standard/relay.py | 50 ++++++++++--- app/nullius/standard/run.py | 6 +- app/static/builder/command-builder-core.js | 24 +++++-- app/static/builder/command-builder.js | 23 ++++-- app/tests/test_standard_recipe_served_by.py | 26 +++++++ app/tests/test_standard_served_by.py | 77 ++++++++++++++++++++- app/ui_tests/test_served_by_builder.py | 53 +++++++++++++- app/web/builder.py | 3 +- 12 files changed, 278 insertions(+), 56 deletions(-) diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index 0f51295..3bea88b 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -716,8 +716,11 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] + # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's + sends = {model} | set(routes.values()) | {str(s).partition("=")[2] or str(s) for s in side_models} for m, slugs in sorted((serving or {}).items()): - a += ["--served-by", f"{m}={','.join(slugs)}"] + if m in sends: + a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] return a diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index ee91a09..d1a89f3 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -401,8 +401,9 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: def _named(name: str) -> str: - """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``Google AI Studio`` - reads ``google-ai-studio``) and nothing else folded: ``Prov/A`` is not ``prov-a``. A run fetches no list of + """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``DeepInfra`` reads + ``deepinfra``) and nothing else folded: ``Prov/A`` is not ``prov-a``, and a provider whose name differs from its + slug otherwise (``Google`` serving ``google-vertex``) reads as another. A run fetches no list of providers, and this is all a name is matched by.""" return re.sub(r"\s+", "-", name.strip().lower()) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 010224d..9d9c315 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -66,10 +66,18 @@ def _at(d, path): def _served(shards: List[dict], results: List[dict]) -> Optional[dict]: - """The merged job's ``served`` (``job.served_total``). A model the shards pinned differently - merged - with ``force`` - reads ``unknown``: the merged config keeps the first shard's pin, which the others' calls - were never held to.""" - who = job.served_total(results) + """The merged job's ``served`` (``job.served_total``), made of each shard's whole one - its trials' and its + benchmark relay's calls, an ``unknown`` kept - else, for a shard that has none, of its trials'. A model the + shards pinned differently - merged with ``force`` - reads ``unknown``: the merged config keeps the first + shard's pin, which the others' calls were never held to.""" + parts = [] + for s in shards: + whole = (s.get("result") or {}).get("served") + if isinstance(whole, dict): + parts.append(whole) + else: + parts.append(job.served_total([r for r in results if (r.get("standard") or {}).get("shard_job") == s.get("name")])) + who = job.served_total([{"standard": {"served": p}} for p in parts if p]) pins = [_at(s["config"], ("standard", "served_by")) or {} for s in shards] pins = [p if isinstance(p, dict) else {} for p in pins] differ = sorted(m for m in {m for p in pins for m in p} if len({json.dumps(p.get(m)) for p in pins}) > 1) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 53b602b..f13a227 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -291,7 +291,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) - out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""} + out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "", + "served_models": {}} name = str(recipe.get("name") or "") if not _NAME.match(name): add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.") @@ -512,30 +513,39 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("limits", f"A spending cap needs a price for {', '.join(missing)}: update the price list in Settings, or set a price there.") judge_def, sim_def = (b.get("score") or {}).get("judge"), b.get("simulator") - def pinned(p: dict, used: Optional[set] = None) -> Dict[str, List[str]]: - """The --served-by of one job: each pinned role's model as the run sends it, where the job calls it - (ledger.models, as the run reads them); ``used`` collects the roles that apply.""" - sent = {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, + def sent_of(p: dict) -> Dict[str, Optional[str]]: + """Each model field's model as job ``p`` sends it; None where it has none (a control has no model of its own).""" + return {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, "judge": (judge_model or providers.served_as(judge_def["model"], provider)) if judge_def else None, "simulator": (sim_model or providers.served_as(sim_def["model"], provider)) if sim_def else None} - called = ledger.models(p["o"], sent["model"], sent["judge"], sent["simulator"], None, control=p["control"]) - got: Dict[str, List[str]] = {} + + def called_of(p: dict) -> List[str]: + s = sent_of(p) + return ledger.models(p["o"], s["model"], s["judge"], s["simulator"], None, control=p["control"]) + + # the pins by model, as the run sends each: a field's pin is its model's, on every job that calls that model + # (a control's judge job too, when the judge's model is the one the model field pinned) + by_model: Dict[str, List[str]] = {} + for p in picks: for role, slugs in served.items(): - m = sent.get(role) - if not m or m not in called: + m = sent_of(p).get(role) + if not m: continue - if used is not None: - used.add(role) - if m in got and got[m] != list(slugs): + if m in by_model and by_model[m] != list(slugs): add("model", f"Served by pins {m} twice, with different providers: give its fields the same ones.") - got[m] = list(slugs) - return got + by_model[m] = list(slugs) + # each field's model as the run sends it, for the builder to ask its providers under that name + out["served_models"] = {role: next((sent_of(p)[role] for p in sorted(picks, key=lambda q: q["control"]) if sent_of(p)[role]), None) + for role in roles} + + def pinned(p: dict) -> Dict[str, List[str]]: + """The --served-by of one job: the pins of the models it calls.""" + called = called_of(p) + return {m: s for m, s in by_model.items() if m in called} if served and not errors: - used: set = set() - for p in picks: - pinned(p, used) + everything = {m for p in picks for m in called_of(p)} for role in served: - if role not in used: + if not {sent_of(p)[role] for p in picks} & everything: add("model", f"Served by: no job of this recipe calls {roles[role]}; leave it empty.") if errors: return out diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 231b184..7cb4908 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -273,9 +273,11 @@ def parse_served_by(given, model: str) -> tuple: def _clean(value, secrets=()): """A recorded copy, never what is sent: a value under a credential's name - replaced, the relay's own key cut out of every string, then ``redact``.""" + replaced, the relay's own key cut out of every string - a key of a + mapping as much as a value - then ``redact``.""" if isinstance(value, dict): - return {k: "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) for k, v in value.items()} + return {_clean(k, secrets): "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) + for k, v in value.items()} if isinstance(value, list): return [_clean(v, secrets) for v in value] if isinstance(value, str): @@ -291,19 +293,23 @@ def _name(v) -> Optional[str]: def _an_error(e) -> Optional[dict]: - """An error object, in one shape: its code, type and message, as given.""" + """An error object, in one shape: its code (a word or a number), type and + message (words), as given; None when it has none of them of that kind. + Nothing else of the object is kept.""" if not isinstance(e, dict) or not e: return None got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} if isinstance(e.get("message"), str): got["message"] = _cut(e["message"], 500) - return got or {"message": _cut(json.dumps(e), 500)} + return got or None def _provider(obj) -> Optional[str]: """The serving provider an object of a reply names: its own ``provider``, its ``message``'s or its ``response``'s, else the last attempt of - OpenRouter's routing facts (``openrouter_metadata``) there.""" + OpenRouter's routing facts (``openrouter_metadata``) there - the one that + answered; an earlier, failed attempt is never taken for it, and a last + attempt that names none gives none.""" if not isinstance(obj, dict): return None inner = [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)] @@ -313,12 +319,20 @@ def _provider(obj) -> Optional[str]: for o in inner: meta = o.get("openrouter_metadata") attempts = meta.get("attempts") if isinstance(meta, dict) else None - named = [a["provider"] for a in attempts if isinstance(a, dict) and _name(a.get("provider"))] if isinstance(attempts, list) else [] - if named: - return named[-1] + if isinstance(attempts, list) and attempts: + last = attempts[-1] + return _name(last.get("provider")) if isinstance(last, dict) else None return None +def _attempts_in(obj) -> bool: + """Whether an object of a reply holds OpenRouter's routing facts, with at least one attempt.""" + return isinstance(obj, dict) and any( + isinstance(o, dict) and isinstance(o.get("openrouter_metadata"), dict) + and isinstance(o["openrouter_metadata"].get("attempts"), list) and bool(o["openrouter_metadata"]["attempts"]) + for o in (obj, obj.get("message"), obj.get("response"))) + + def _choices(choices, ends: dict, errors: dict) -> None: """Each choice's end and error, by its index.""" if not isinstance(choices, list): @@ -365,7 +379,9 @@ def _facts(path: str, body: bytes, streamed: bool) -> dict: continue if not isinstance(ev, dict): continue - served = served or _provider(ev) + # the latest event that names one: the end's over an earlier attempt's; routing facts whose last + # attempt names none leave none + served = _provider(ev) if _attempts_in(ev) else (_provider(ev) or served) kind = ev.get("type") if kind == "error": # Messages' error event, or a Responses one error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) @@ -969,11 +985,23 @@ def serve(argv=None) -> int: ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", - help="the serving providers that may answer a request sent as MODEL, in order, and no other") + help="the serving providers that may answer a request sent as MODEL, in order, and no other; " + "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " + "or --routing given") + ap.add_argument("--routing", action="store_true", + help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") a = ap.parse_args(argv) serving, bad = parse_served_by(a.served_by, "") + if serving and not bad: + host = (urllib.parse.urlsplit(a.base).hostname or "").lower() + if not (a.routing or host == "openrouter.ai" or host.endswith(".openrouter.ai")): + bad.append(f"{a.base} routes no model to several serving providers (OpenRouter's does; --routing says another does)") + sent = {a.model} | {r.partition("=")[2] for r in a.route} | {(s.partition("=")[2] or s) for s in a.side_model} + stray = [m for m in serving if a.model and m not in sent] + if stray: + bad.append(f"{', '.join(stray)}: a model this relay does not send (it sends {', '.join(sorted(x for x in sent if x))})") if bad: - print(json.dumps({"error": "; ".join(bad)}), flush=True) + print(json.dumps({"error": "--served-by: " + "; ".join(bad)}), flush=True) return 2 try: with open(a.key_file) as f: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 6ce474c..07f3721 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -506,9 +506,9 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas if not PROVIDERS.get(provider, {}).get("routing"): raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " "among; the option needs a provider that routes a model to several (openrouter)") - # a control calls no model of its own: only its judge, simulated user or an option's models - called = [m for m in self.models_called() - if other.get("_kind") != "control" or m in (self.judge_model, self.simulator_model, self.embed_model, self.rerank_model)] + # a control calls no model of its own and plays no user: only its judge and an option's models + called = ([m for m in (self.judge_model, self.embed_model, self.rerank_model) if m] if other.get("_kind") == "control" + else self.models_called()) stray = [m for m in self.serving if m not in called] if stray: raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 85b1943..395952f 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -69,6 +69,7 @@ r.model_choice = choice; if (paperListed(r)) r.model = ''; if (choice !== 'any') { delete r.judge_model; delete r.simulator_model; } + if (choice !== was) delete r.served_by; // its models change with it: a pin belongs to the model it was set for if (lb && was === 'leaderboard' && choice !== 'leaderboard') { r.bench_options = {...(r.bench_options || {})}; for (const k of Object.keys(lb.bench_options || {})) delete r.bench_options[k]; @@ -82,14 +83,24 @@ // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. const SERVED_ROLES = ['model', 'simulator', 'judge']; - const SERVED_SLUG = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9._-]+)*$/; function servedRoles(r) { const b = standardBench(r); if (!b || r.provider !== 'openrouter' || standardKeyless(r)) return []; return SERVED_ROLES.filter(k => k === 'model' || (k === 'simulator' && b.simulator) || (k === 'judge' && b.judge)); } - // A field's text, "deepinfra, together", as the pin's list; an empty field pins nothing. - const servedList = text => String(text || '').split(',').map(s => s.trim()).filter(Boolean); + // A field's text, "deepinfra, together", as the pin's list, every part kept for the server to check ("a,,b" is + // refused in sight, as the option refuses it); a field with nothing in it pins nothing. + const servedList = text => { const parts = String(text || '').split(',').map(s => s.trim()); return parts.some(Boolean) ? parts : []; }; + // A model field's role, and its model changed: the pin set for the model before it goes with it. + const SERVED_FIELD = {model: 'model', simulator_model: 'simulator', judge_model: 'judge'}; + function setModelOf(r, field, value) { + if (SERVED_FIELD[field] && r[field] !== value && r.served_by) setServed(r, SERVED_FIELD[field], []); + r[field] = value; + return r; + } + // The model a field's Served by lists the providers of: the one the run sends, as the server compiled it + // (recipe.py, served_models: a paper model's served name, an official one's with its maker), else the one shown. + const servedModelOf = (role, shown, compiled) => ((compiled || {}).served_models || {})[role] || shown || ''; function setServed(r, role, slugs) { const list = Array.isArray(slugs) ? slugs : servedList(slugs); r.served_by = {...(r.served_by || {})}; @@ -112,7 +123,7 @@ const b = standardBench({benchmark: key}); if (!b) return r; const was = standardBench(r); - if (was !== b) r.model_choice = 'any'; + if (was !== b) { r.model_choice = 'any'; delete r.served_by; } r.benchmark = key; const kept = was === b ? datasetsOf(r).filter(id => b.datasets.some(d => d.id === id)) : []; r.datasets = kept.length ? kept : [b.datasets[0].id]; @@ -162,7 +173,8 @@ // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) for (const role of SERVED_ROLES) { const v = raw.served_by[role]; - if (Array.isArray(v) && v.length && v.every(s => typeof s === 'string' && SERVED_SLUG.test(s.trim()))) setServed(r, role, v.map(s => s.trim())); } + // kept as saved, a slug the server refuses included: a pin is refused in sight, never dropped in silence + if (Array.isArray(v) && v.length) setServed(r, role, v.map(s => typeof s === 'string' ? s.trim() : String(s))); } r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; else r.model = bareModel(r.model); @@ -782,5 +794,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,setModelOf,servedModelOf,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 74fba43..37a3016 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -114,7 +114,9 @@ }; // The standard's recipe as the server compiles it from the definitions (POST // /new-run/runner). Nothing typed here is a command; Run sends the choices. - let runnerOut = null, runnerKey = '', runnerTimer; + let runnerOut = null, runnerKey = '', runnerTimer, runnerOutKey = ''; + // the server's compile of the recipe as it is now, or null while a newer one is asked + const compiledNow = () => runnerOutKey === JSON.stringify(r()) ? runnerOut : null; // Switching runner keeps the recipe being edited, one per runner, for as long as the page is open: switching back finds it as it was. const keptByRunner = {}; // Nullius's own mark for the standard; Harbor's for Harbor @@ -635,7 +637,8 @@ out = await res.json(); } catch (e) { out = {errors: [{step: REVIEW, message: 'The recipe could not be compiled here: ' + e}], notes: [], review: []}; } if (key !== runnerKey) return; - runnerOut = out; + runnerOut = out; runnerOutKey = key; + if (step === S.model) fillServed(); if (step === REVIEW) renderForm(); refreshPlan(); }, 120); @@ -647,10 +650,10 @@ // Served by: under a model field, on OpenRouter alone, the serving providers that may answer it, in order (recipe.py, // served_by); typed as slugs, or added from the model's providers (POST /new-run/endpoints) const SERVED_NAME = {model: 'model', simulator: 'user simulator', judge: 'judge'}; + // its list: the providers of the model the run sends (the server's served_models), not of the name shown const servedField = (rec, role, model) => !C.servedRoles(rec).includes(role) ? '' : - `
` - + `` - + `OpenRouter's providers for this model, tried in this order and no other. Empty: any of them.
`; + `
` + + `
`; const servedCache = new Map(); function servedList(model) { let entry = servedCache.get(model); @@ -664,6 +667,11 @@ } function fillServed() { root.querySelectorAll('select[data-served-add]').forEach(async select => { + // the model the run sends, once the server has compiled it: a list asked under another name is asked again + const sent = C.servedModelOf(select.dataset.servedAdd, select.dataset.servedShown, compiledNow()); + if (sent !== select.dataset.servedModel) { select.dataset.servedModel = sent; select.length = 1; delete select.dataset.servedFilled; } + if (select.dataset.servedFilled) return; + select.dataset.servedFilled = 'true'; const model = select.dataset.servedModel; if (!model) return; const got = await servedList(model); @@ -987,6 +995,7 @@ else if(!d.providerModel&&['model','userModel','evalModel'].includes(d.field)&&/^(openrouter|openai)\//.test(String(value))){rec.provider=C.providerOf(rec,value);rec[d.field]=C.bareModel(value); renderForm();refreshPlan();const f=root.querySelector(`[data-field="${d.field}"]`);if(f){f.focus();f.setSelectionRange(f.value.length,f.value.length);}return;} // the provider the prefix named, and whether it is set else if(d.field==='provider'){C.setChoice(rec,value);renderForm();refreshPlan();return;} // whether it is set in Settings, and the names a runner's models are sent as, follow the provider + else if(['model','simulator_model','judge_model'].includes(d.field))C.setModelOf(rec,d.field,value); // the field's pin goes with its model else rec[d.field]=value; if(el.list?.id==='nb-paths')completeDirectory(el.value); }else if(d.minutes){C.setPhaseMinutes(rec,d.minutes,el.value===''?'':Number(el.value));const row=el.closest('tr');if(row){const m=row.querySelector('.nb-mono');const t=C.effectiveTimeouts(rec);const p=t&&t.phases.find(x=>x.key===d.minutes);if(m&&p)m.textContent='ร—'+p.multiplier;}refreshPlan();return;} @@ -1011,7 +1020,9 @@ // a provider added to a Served by, after the ones already there; the field shows the list, in its order else if(d.servedAdd!==undefined){const rec=r(),role=d.servedAdd,now=(rec.served_by||{})[role]||[];if(el.value&&!now.includes(el.value))C.setServed(rec,role,[...now,el.value]);el.value='';const f=root.querySelector(`[data-served-by="${role}"]`);if(f)f.value=((r().served_by||{})[role]||[]).join(', ');refreshPlan();} // a model chosen: its Served by lists that model's providers - else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan();} + // a model chosen: its Served by lists that model's providers; the keyboard stays on the field the new form holds + else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan(); + requestAnimationFrame(()=>{const f=root.querySelector(`[data-field="${d.field}"]`);if(f&&document.activeElement!==f)f.focus();});} else if(d.stdVersion!==undefined){C.setStandardBenchmark(r(),el.value);render();} // an option at its default is left out: the recipe names only what was changed else if(d.benchOptSelect!==undefined){const rec=r(),o=((C.standardBench(rec)||{}).options||{})[d.benchOptSelect]||{};rec.bench_options={...(rec.bench_options||{})};const v=el.value.trim();if(!v||v===o.default)delete rec.bench_options[d.benchOptSelect];else rec.bench_options[d.benchOptSelect]=v;refreshPlan();} diff --git a/app/tests/test_standard_recipe_served_by.py b/app/tests/test_standard_recipe_served_by.py index d1a3a70..a24a99e 100644 --- a/app/tests/test_standard_recipe_served_by.py +++ b/app/tests/test_standard_recipe_served_by.py @@ -88,6 +88,32 @@ def test_one_model_in_two_fields_takes_one_list(self): other = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["b"]}), d) self.assertIn("pins openai/gpt-4o-mini twice", other["errors"][0]["message"]) + def test_a_pin_is_its_models_on_every_job_that_calls_it(self): + """The model field's model is also the judge's: the control beside the agent calls it as its judge, and its job + takes the pin too.""" + out = recipe.compile(a_recipe(others=["fake", "nop"], model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), defs(judged())) + self.assertEqual(out["errors"], []) + for part in out["parts"]: + self.assertIn("--served-by openai/gpt-4o-mini=prov-a", part["script"]) + + def test_a_list_with_an_empty_or_a_bad_slug_is_refused_in_sight(self): + for slugs in (["deepinfra", ""], ["deepinfra", "bad slug"]): + with self.subTest(slugs=slugs): + out = recipe.compile(a_recipe(served_by={"model": slugs}), defs()) + self.assertTrue(any("not a serving provider's slug" in e["message"] for e in out["errors"]), out["errors"]) + + def test_each_fields_model_is_given_as_the_run_sends_it(self): + """Like the paper, LongMemEval's gpt-4o is sent as openai/gpt-4o-2024-08-06 on OpenRouter: its providers are that + model's, and the pin is put on that name.""" + sets = definitions.find(definitions.load("benchmark")[0], "longmemeval")["datasets"] + r = a_recipe(benchmark="longmemeval", dataset=sets[0]["id"], other="", others=["original"], model_choice="paper", model="gpt-4o", + served_by={"model": ["openai"]}) + out = recipe.compile(r) + sent = out["served_models"]["model"] + self.assertEqual(sent, "openai/gpt-4o-2024-08-06") + self.assertIn(f"--served-by {sent}=openai", out["script"]) + self.assertEqual(recipe.compile(a_recipe(), defs())["served_models"]["model"], "openai/gpt-5-mini") + def test_a_recipe_with_no_pin_compiles_to_the_command_it_did(self): plain = recipe.compile(a_recipe(), defs()) self.assertEqual(plain["errors"], []) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index 7c1fb24..2bbb9c8 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -177,7 +177,7 @@ def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): for serving in (None, {"m-run": ["prov-a"]}): with self.subTest(serving=serving): r = self.relay(model="m-run", key=KEY, serving=serving) - asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}"}) + asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}", KEY: "as a key"}) post(r.url("u0") + "/chat/completions", asked) c = r.calls("u0")[-1] self.assertNotIn(KEY, json.dumps(c)) @@ -368,7 +368,7 @@ def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): f.write("or-test") p = subprocess.Popen([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", "--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, "--model", "m-run", - "--served-by", "m-run=prov-a,prov-b/fp8"], + "--served-by", "m-run=prov-a,prov-b/fp8", "--routing"], cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) try: ready, _, _ = select.select([p.stdout], [], [], 20) # its one line, or no more than 20 s @@ -583,6 +583,79 @@ def test_answered_calls_that_name_no_provider_are_counted(self): self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) +class TheReview(unittest.TestCase): + """What a review of the pin found, each a rule of its own.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_the_provider_of_the_replys_end_is_recorded_never_an_earlier_attempts(self): + events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": { + "attempts": [{"provider": "Prov A", "status": 503}]}}}, + {"type": "response.completed", "response": {"status": "completed", "output": [], "provider": "Prov B"}}] + facts = relay.reply_facts("/responses", sse(*events), True) + held = job.served([{"status": 200, "model": "m", "reply": facts}], {"m": ["prov-a"]})["held"]["m"] + self.assertEqual((facts["served_by"], held), ("Prov B", "unknown")) + + def test_a_last_attempt_that_names_no_provider_takes_no_earlier_name(self): + body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"attempts": [ + {"provider": "Prov A", "status": 503}, {"provider": 7, "status": 200}]}} + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) + stream = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "openrouter_metadata": {"attempts": [{"provider": "Prov A", "status": 503}, {"status": 200}]}}) + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", stream, True)) + + def test_an_error_keeps_its_stated_fields_alone(self): + nested = relay.reply_facts("/chat/completions", json.dumps({"error": {"api_key": ["review-only-credential"]}}).encode(), False) + self.assertNotIn("review-only-credential", json.dumps(nested)) + self.assertEqual(relay.reply_facts("/messages", b'{"type":"message","content":[],"error":{"code":[],"type":{},"message":17}}', False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b'{"error":{"code":502,"message":"m","detail":{"key":"x"}}}', False), + {"error": {"code": 502, "message": "m"}}) + + def test_of_several_choices_errors_the_top_level_one_else_the_first_is_kept(self): + two = [{"index": 0, "finish_reason": "error", "error": {"code": 429, "message": "rate limited"}}, + {"index": 1, "finish_reason": "error", "error": {"code": 503, "message": "unavailable"}}] + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps({"choices": two}).encode(), False)["error"], + {"code": 429, "message": "rate limited", "choice": 0}) + top = {"choices": two, "error": {"code": 500, "message": "whole"}} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(top).encode(), False)["error"], {"code": 500, "message": "whole"}) + + def test_a_merged_job_keeps_each_shards_whole_summary(self): + """A shard's summary holds its benchmark relay's calls too: a merge made of its trials alone would drop + them, and an ``unknown`` would read true.""" + pin = {"m": ["prov-a"]} + trial = {"standard": {"served": job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov A"}}], pin), "shard_job": "s1"}} + bench = job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov B"}}], pin) + shard = {"name": "s1", "config": {"standard": {"served_by": pin}}, "result": {"served": job.served_total([trial], bench)}} + self.assertEqual(merge._served([shard], [trial]), shard["result"]["served"]) + old = {"name": "s2", "config": {"standard": {"served_by": pin}}, "result": {}} # a shard written before: its trials + self.assertEqual(merge._served([old], [dict(trial, standard=dict(trial["standard"], shard_job="s2"))])["held"], {"m": True}) + + def test_a_control_is_refused_a_pin_on_its_simulated_users_model(self): + b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing"}}, + simulator={"model": "sim"}) + with self.assertRaisesRegex(run.Refused, "names sim, which this run does not call"): + run.Run(b, definitions.control(b, "nop"), model="sim", provider="openrouter", env={}, + jobs_root=self.tmp, say=lambda _: None, served_by={"sim": ["prov-a"]}) + + def test_the_relays_own_command_refuses_another_provider_and_a_model_it_does_not_send(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("review-test-key") + for base, model, extra in (("https://api.openai.com/v1", "m", []), ("https://openrouter.ai/api/v1", "unused", []), + ("http://127.0.0.1:9/v1", "m", [])): + with self.subTest(base=base, model=model): + out = io.StringIO() + with mock.patch.object(relay, "Relay") as fake, redirect_stdout(out): + code = relay.serve(["--base", base, "--model", "m", "--key-file", key, "--record", self.tmp, + "--served-by", model + "=prov-a", *extra]) + self.assertEqual(code, 2) + self.assertIn("--served-by", json.loads(out.getvalue())["error"]) + fake.assert_not_called() + + class TheMerge(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() diff --git a/app/ui_tests/test_served_by_builder.py b/app/ui_tests/test_served_by_builder.py index 52ac3c3..11e82fa 100644 --- a/app/ui_tests/test_served_by_builder.py +++ b/app/ui_tests/test_served_by_builder.py @@ -5,6 +5,7 @@ import json import os import unittest +from pathlib import Path from unittest.mock import patch from .builder_support import CompilerCase @@ -59,6 +60,30 @@ def test_guards_and_another_provider_do_not_fetch(self): fetch.assert_not_called() +class TheStep(unittest.TestCase): + """What the Model step's script draws and what the compile route hands it.""" + SCRIPT = (Path(__file__).resolve().parents[1] / 'static' / 'builder' / 'command-builder.js').read_text() + + def test_served_by_holds_no_sentence_and_its_help_is_a_tooltip(self): + field = self.SCRIPT[self.SCRIPT.index('const servedField'):self.SCRIPT.index('const servedCache')] + self.assertNotIn('', field) + self.assertIn('title="', field) + + def test_the_compile_route_gives_each_fields_model_as_the_run_sends_it(self): + client = web.app.test_client() + with client.session_transaction() as session: + session['csrf'] = 'served-test-token' + recipe = {'version': 1, 'harness': 'standard', 'name': 'x-0101', 'title': 'x', 'benchmark': 'longmemeval', + 'datasets': ['longmemeval_s'], 'others': ['original'], 'model_choice': 'paper', 'provider': 'openrouter', + 'model': 'gpt-4o', 'limit': 1, 'concurrency': 1, 'where': 'local'} + response = client.post('/new-run/runner', json={'recipe': recipe}, headers={'X-CSRF-Token': 'served-test-token'}) + self.assertEqual(response.json['served_models']['model'], 'openai/gpt-4o-2024-08-06') + + def test_the_keyboard_stays_on_the_model_field_once_the_form_is_drawn_again(self): + self.assertIn("requestAnimationFrame(()=>{const f=root.querySelector(`[data-field=\"${d.field}\"]`);if(f&&document.activeElement!==f)f.focus();})", + self.SCRIPT) + + class ServedInTheCore(CompilerCase): """command-builder-core.js: which model fields take a Served by, and the recipe's ``served_by``.""" @@ -76,10 +101,34 @@ def test_a_saved_recipe_keeps_its_pins_and_another_provider_drops_them(self): 'served_by': {'model': ['deepinfra/fp8', ' together '], 'judge': ['bad slug'], 'agent': ['x'], 'simulator': []}} out = self.js('const b=C.standard().benchmarks[0];const raw=' + json.dumps(raw) + ';raw.benchmark=b.key;' 'const r=C.fromJSON(JSON.stringify(raw));const kept=JSON.parse(JSON.stringify(r.served_by||null));' - 'C.setServed(r,"model","a, b ,");const typed=r.served_by.model;C.setServed(r,"model","");const emptied=r.served_by===undefined;' + 'C.setServed(r,"model","a, b");const typed=r.served_by.model;C.setServed(r,"model","");const emptied=!(r.served_by||{}).model;' 'C.setServed(r,"model",["a"]);C.setChoice(r,"openai");' 'process.stdout.write(JSON.stringify([kept,typed,emptied,r.served_by===undefined]));') - self.assertEqual(out, [{'model': ['deepinfra/fp8', 'together']}, ['a', 'b'], True, True]) + # kept as written, the judge's bad slug included: the server refuses it in sight (test_a_pin_is_kept_as_written...) + self.assertEqual(out, [{'model': ['deepinfra/fp8', 'together'], 'judge': ['bad slug']}, ['a', 'b'], True, True]) + + def test_a_pin_goes_with_the_model_it_was_set_for(self): + out = self.js('const b=C.standard().benchmarks.find(x=>x.judge)||C.standard().benchmarks[0];const r=C.createStandard(b.key);C.setChoice(r,"openrouter");' + 'const seen=[];C.setServed(r,"model",["a"]);C.setServed(r,"judge",["j"]);' + 'C.setModelOf(r,"model",r.model);seen.push(JSON.stringify(r.served_by));' + 'C.setModelOf(r,"model","meta-llama/llama-3.3-70b-instruct");seen.push(JSON.stringify(r.served_by));' + 'C.setModelOf(r,"judge_model","openai/gpt-4o");seen.push(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model",["a"]);C.setModelChoice(r,r.model_choice);seen.push(JSON.stringify(r.served_by));' + 'C.setModelChoice(r,C.paperOffered(r)?"paper":"leaderboard");seen.push(JSON.stringify(r.served_by||null));' + 'C.setServed(r,"model",["a"]);const other=C.standard().benchmarks.find(x=>x.key!==b.key);C.setStandardBenchmark(r,other.key);seen.push(JSON.stringify(r.served_by||null));' + 'process.stdout.write(JSON.stringify(seen));') + self.assertEqual(out, ['{"model":["a"],"judge":["j"]}', '{"judge":["j"]}', 'null', '{"model":["a"]}', 'null', 'null']) + + def test_a_pin_is_kept_as_written_for_the_server_to_refuse(self): + raw = {'version': 1, 'harness': 'standard', 'benchmark': '', 'provider': 'openrouter', 'served_by': {'model': ['deepinfra', 'bad slug']}} + out = self.js('const raw=' + json.dumps(raw) + ';raw.benchmark=C.standard().benchmarks[0].key;const r=C.fromJSON(JSON.stringify(raw));' + 'process.stdout.write(JSON.stringify([r.served_by,C.servedList("a,,b"),C.servedList(" , "),C.servedList("")]));') + self.assertEqual(out, [{'model': ['deepinfra', 'bad slug']}, ['a', '', 'b'], [], []]) + + def test_the_list_is_of_the_model_the_run_sends(self): + out = self.js('process.stdout.write(JSON.stringify([C.servedModelOf("model","gpt-4o",{served_models:{model:"openai/gpt-4o-2024-08-06"}}),' + 'C.servedModelOf("judge","gpt-4o",null),C.servedModelOf("judge","gpt-4o",{served_models:{judge:null}})]));') + self.assertEqual(out, ['openai/gpt-4o-2024-08-06', 'gpt-4o', 'gpt-4o']) def test_a_recipe_with_no_pin_is_saved_as_before(self): out = self.js('const b=C.standard().benchmarks[0];const r=C.createStandard(b.key);C.setChoice(r,"openrouter");' diff --git a/app/web/builder.py b/app/web/builder.py index e069744..78e4daf 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -92,7 +92,8 @@ def new_run_runner(): abort(413) if recipe.get("harness") == standard_recipe.HARNESS: out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, unpriced=execute.unpriced) - return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm")} + # served_models: each model field's model as the run sends it, which Served by lists the providers of + return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm", "served_models")} return {"errors": [{"step": providers.STEP["runner"], "message": providers.GONE}], "notes": [], "review": []}, 404 From 63d11c89a3fe72010efccff30c51295fbba5f4b7 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:40:55 +0100 Subject: [PATCH 12/24] docs(standard): say what a slug's variant is, which error is kept, and where held reads unknown A variant may be a quantisation, a speed or a region. Of several choices' errors the record keeps the top-level one, else the first choice's. A provider whose name is not its slug once lowercased reads unknown though it served the call; the example of a name that holds is DeepInfra. --- docs/cli/run.md | 10 +++++----- docs/standard.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index def6026..ee86bf4 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -76,14 +76,14 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | Rule | Detail | | --- | --- | -| A slug | As the model's page on OpenRouter names its provider: `deepinfra`, `deepinfra/fp8`. Several are tried in their order, and no other provider answers | +| A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | | Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | | What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`), where its reply names the provider that served it. A model with no pin is sent as before | -| Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | +| Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | | No provider can serve it | OpenRouter answers with an error and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | -| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`Google AI Studio` is `google-ai-studio`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | -| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra` | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider of the reply's end (an earlier, failed attempt's never), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | +| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A provider whose name is not its slug once lowercased reads `unknown` though it served the call: `DeepInfra` is `deepinfra`, and `Google` is not `google-vertex` | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | | The builder | *Model* step, **Served by** under each model field, on OpenRouter alone: the recipe's `served_by`, by field (`model`, `simulator`, `judge`), compiled into `--served-by` for each job that calls that model. Its list of a model's providers comes from OpenRouter (`/models/{maker}/{model}/endpoints`), asked as the model list is; when it cannot be had, a slug typed by hand is taken | diff --git a/docs/standard.md b/docs/standard.md index 83c736d..6faa4fd 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -147,7 +147,7 @@ record); verifier/reward.txt and score.json | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | -| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | | Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | From 980f22d784cf7ba3a91894708e0b02f706facd4d Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:47:31 +0100 Subject: [PATCH 13/24] fix(standard): read who served a reply from the endpoint its routing facts mark as selected OpenRouter's routing facts list the endpoints available for a request and mark the one that served it as selected; they hold no list of attempts. The relay now reads that entry, at the top of a reply and in a completed Responses event's response, and asks for the facts on a pinned Responses request too: a Responses reply names its provider there alone, so a pin on a model an agent calls through Responses could never read as held. --- app/nullius/standard/relay.py | 62 ++++++++-------- app/tests/test_standard_served_by.py | 107 ++++++++++++++++++++------- 2 files changed, 111 insertions(+), 58 deletions(-) diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 7cb4908..78c9c06 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -304,33 +304,34 @@ def _an_error(e) -> Optional[dict]: return got or None +def _routing_facts(obj) -> Optional[list]: + """The serving endpoints OpenRouter's routing facts (``openrouter_metadata``) + list as available, in an object of a reply, its ``message`` or its + ``response``; None when none of them holds such a list.""" + if not isinstance(obj, dict): + return None + for o in (obj, obj.get("message"), obj.get("response")): + meta = o.get("openrouter_metadata") if isinstance(o, dict) else None + endpoints = meta.get("endpoints") if isinstance(meta, dict) else None + available = endpoints.get("available") if isinstance(endpoints, dict) else None + if isinstance(available, list): + return available + return None + + def _provider(obj) -> Optional[str]: """The serving provider an object of a reply names: its own ``provider``, - its ``message``'s or its ``response``'s, else the last attempt of - OpenRouter's routing facts (``openrouter_metadata``) there - the one that - answered; an earlier, failed attempt is never taken for it, and a last - attempt that names none gives none.""" + its ``message``'s or its ``response``'s, else the one endpoint OpenRouter's + routing facts mark ``selected`` among the available ones (asked for with + ``X-OpenRouter-Metadata``; a Responses reply names its provider there + alone). Facts that mark no single named endpoint give none.""" if not isinstance(obj, dict): return None - inner = [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)] - for o in inner: + for o in [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)]: if _name(o.get("provider")): return o["provider"] - for o in inner: - meta = o.get("openrouter_metadata") - attempts = meta.get("attempts") if isinstance(meta, dict) else None - if isinstance(attempts, list) and attempts: - last = attempts[-1] - return _name(last.get("provider")) if isinstance(last, dict) else None - return None - - -def _attempts_in(obj) -> bool: - """Whether an object of a reply holds OpenRouter's routing facts, with at least one attempt.""" - return isinstance(obj, dict) and any( - isinstance(o, dict) and isinstance(o.get("openrouter_metadata"), dict) - and isinstance(o["openrouter_metadata"].get("attempts"), list) and bool(o["openrouter_metadata"]["attempts"]) - for o in (obj, obj.get("message"), obj.get("response"))) + chosen = [a for a in _routing_facts(obj) or () if isinstance(a, dict) and a.get("selected") is True] + return _name(chosen[0].get("provider")) if len(chosen) == 1 else None def _choices(choices, ends: dict, errors: dict) -> None: @@ -379,9 +380,8 @@ def _facts(path: str, body: bytes, streamed: bool) -> dict: continue if not isinstance(ev, dict): continue - # the latest event that names one: the end's over an earlier attempt's; routing facts whose last - # attempt names none leave none - served = _provider(ev) if _attempts_in(ev) else (_provider(ev) or served) + # the latest event that names one; routing facts that mark no single named endpoint leave none + served = _provider(ev) if _routing_facts(ev) is not None else (_provider(ev) or served) kind = ev.get("type") if kind == "error": # Messages' error event, or a Responses one error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) @@ -583,9 +583,9 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s and not sent when it does not fit. ``serving``: by model, the serving providers that may answer a request sent as that model (:func:`routing`), put on every such request in place of the caller's own. ``facts``: a - pinned Chat Completions request asks OpenRouter for its routing facts, - where its reply names who served it; by default when ``base`` is - OpenRouter's.""" + pinned Chat Completions or Responses request asks OpenRouter for its + routing facts, where its reply names who served it; by default when + ``base`` is OpenRouter's.""" self.fields = fields self.serving = {m: list(s) for m, s in (serving or {}).items()} host = (urllib.parse.urlsplit(base).hostname or "").lower() @@ -653,8 +653,9 @@ def _said(self, e: Exception) -> str: return redact(text)[:200] def _headers(self, caller, facts: bool = False) -> Dict[str, str]: - """``facts``: a pinned Chat Completions request to OpenRouter, sent - asking for its routing facts whatever the caller said of them.""" + """``facts``: a pinned Chat Completions or Responses request to + OpenRouter, sent asking for its routing facts whatever the caller + said of them.""" out = forwarded(caller) if facts: out = {k: v for k, v in out.items() if k.lower() != "x-openrouter-metadata"} @@ -724,7 +725,8 @@ def _forward(self, h: BaseHTTPRequestHandler): conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed try: - facts = bool(pinned) and self.facts and rest.split("?")[0].endswith("/chat/completions") + # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself + facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index 2bbb9c8..8aa3690 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -40,7 +40,18 @@ def sse(*events) -> bytes: return b"".join(b"data: " + json.dumps(e).encode() + b"\n\n" for e in events) + b"data: [DONE]\n\n" -# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions, Messages, Responses. +def routing_facts(selected, *others) -> dict: + """OpenRouter's routing facts (``openrouter_metadata``) as a reply holds them when asked: the endpoint that + served it is the one marked ``selected`` among the available ones.""" + available = [{"provider": o, "model": "asked", "selected": False} for o in others] + if selected is not None: + available.insert(len(available) // 2, {"provider": selected, "model": "asked", "selected": True}) + return {"requested": "asked", "attempt": 1, "summary": f"available={len(available)}, selected={selected}", + "endpoints": {"total": len(available) + 2, "available": available}} + + +# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions and Messages of +# themselves; Responses in the routing facts alone, given when the request asks for them (``ASKED``). CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 3, "completion_tokens": 1}} CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "h"}, "finish_reason": None}]}, @@ -52,17 +63,22 @@ def sse(*events) -> bytes: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 1}}, {"type": "message_stop"}) -RESPONSES = {"object": "response", "provider": "Prov C", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, +RESPONSES = {"object": "response", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, "output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}], "usage": {"input_tokens": 3, "output_tokens": 1}} -RESPONSES_SSE = sse({"type": "response.created", "response": {"provider": "Prov C", "status": "in_progress"}}, +COMPLETED = {"object": "response", "status": "completed", "output": [], "usage": {"input_tokens": 3, "output_tokens": 1}} +RESPONSES_SSE = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, {"type": "response.output_text.delta", "delta": "hi"}, - {"type": "response.completed", "response": {"provider": "Prov C", "status": "completed", "output": [], - "usage": {"input_tokens": 3, "output_tokens": 1}}}) + {"type": "response.completed", "response": COMPLETED}) +RESPONSES_ASKED = dict(RESPONSES, openrouter_metadata=routing_facts("Prov C", "Prov D", "Prov E")) +RESPONSES_SSE_ASKED = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov C", "Prov D"))}) +UNASKED = {False: RESPONSES, True: RESPONSES_SSE} SHAPES = {("/chat/completions", False): (CHAT, "Prov A", "stop"), ("/chat/completions", True): (CHAT_SSE, "Prov A", "length"), ("/messages", False): (MESSAGES, "Prov B", "end_turn"), ("/messages", True): (MESSAGES_SSE, "Prov B", "max_tokens"), - ("/responses", False): (RESPONSES, "Prov C", "incomplete: max_output_tokens"), - ("/responses", True): (RESPONSES_SSE, "Prov C", "completed")} + ("/responses", False): (RESPONSES_ASKED, "Prov C", "incomplete: max_output_tokens"), + ("/responses", True): (RESPONSES_SSE_ASKED, "Prov C", "completed")} def body_for(path: str, stream: bool) -> dict: @@ -95,6 +111,8 @@ def do_POST(self): else: path = "/" + self.path.split("/v1/", 1)[-1] reply = SHAPES.get((path, bool(req.get("stream"))), (CHAT,))[0] + if path == "/responses" and self.headers.get("X-OpenRouter-Metadata") != "enabled": + reply = UNASKED[bool(req.get("stream"))] status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ reply if isinstance(reply, bytes) else json.dumps(reply).encode() self.send_response(status) @@ -189,7 +207,7 @@ def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): self.assertEqual(c["request"]["routing_asked"]["token"], "[REDACTED]") def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): - r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) for (path, stream), (_, who, end) in SHAPES.items(): with self.subTest(path=path, stream=stream): post(r.url("u0") + path, body_for(path, stream)) @@ -219,14 +237,18 @@ def test_an_error_beside_a_whole_reply_is_read_in_every_shape(self): self.assertEqual(relay.reply_facts(path, json.dumps(body).encode(), False)["error"], error) def test_a_completed_responses_events_routing_facts_name_who_served_it(self): - body = sse({"type": "response.completed", "response": {"object": "response", "status": "completed", "output": [], - "openrouter_metadata": {"attempts": [{"provider": "Prov A", "status": 200}]}}}) + body = sse({"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov A", "Prov X"))}) self.assertEqual(relay.reply_facts("/responses", body, True), {"served_by": "Prov A", "end": "completed"}) + whole = json.dumps(dict(RESPONSES, openrouter_metadata=routing_facts("Prov A", "Prov X", "Prov Y"))).encode() + self.assertEqual(relay.reply_facts("/responses", whole, False)["served_by"], "Prov A") def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): - odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"attempts": 1}}, + odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"endpoints": {"available": 1}}}, {"type": "message", "content": [], "provider": 7, "error": "a string", "stop_reason": ["x"]}, - {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}] + {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}, + {"choices": [], "openrouter_metadata": {"endpoints": [1]}}, + {"choices": [], "openrouter_metadata": {"endpoints": {"available": [7, {"provider": 7, "selected": True}, + {"provider": "Prov A", "selected": "true"}]}}}] r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) for body in odd: with self.subTest(body=body): @@ -275,15 +297,17 @@ def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): def test_the_routing_facts_are_asked_of_openrouter_alone_and_always_as_enabled(self): meta = json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, - openrouter_metadata={"attempts": [{"provider": "Prov X", "status": 503}, - {"provider": "Prov A", "status": 200}]})).encode() + openrouter_metadata=routing_facts("Prov A", "Prov X"))).encode() r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) Upstream.next = (200, "application/json", meta) post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False), {"X-OpenRouter-Metadata": "disabled"}) got = [v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"] self.assertEqual(got, ["enabled"]) self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") - post(r.url("u0") + "/messages", body_for("/messages", False)) # not a Chat request: nothing asked + for stream in (False, True): # a Responses request: asked too + post(r.url("u0") + "/responses", body_for("/responses", stream), {"X-OpenRouter-Metadata": "disabled"}) + self.assertEqual([v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"], ["enabled"]) + post(r.url("u0") + "/messages", body_for("/messages", False)) # a Messages reply names its provider: nothing asked self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) for kw in ({"serving": {"m-run": ["prov-a"]}}, {"facts": True}): # another provider's address; no pin with self.subTest(kw=kw): @@ -590,22 +614,49 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, True) - def test_the_provider_of_the_replys_end_is_recorded_never_an_earlier_attempts(self): - events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": { - "attempts": [{"provider": "Prov A", "status": 503}]}}}, - {"type": "response.completed", "response": {"status": "completed", "output": [], "provider": "Prov B"}}] + def test_the_selected_endpoint_of_the_replys_end_is_recorded_never_another(self): + """The routing facts list the endpoints available: the one marked selected served the reply, never the + first listed nor one an earlier event's facts marked.""" + events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov B", "Prov A", "Prov C"))}] facts = relay.reply_facts("/responses", sse(*events), True) held = job.served([{"status": 200, "model": "m", "reply": facts}], {"m": ["prov-a"]})["held"]["m"] self.assertEqual((facts["served_by"], held), ("Prov B", "unknown")) - - def test_a_last_attempt_that_names_no_provider_takes_no_earlier_name(self): - body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"attempts": [ - {"provider": "Prov A", "status": 503}, {"provider": 7, "status": 200}]}} - self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) - stream = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, - {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - "openrouter_metadata": {"attempts": [{"provider": "Prov A", "status": 503}, {"status": 200}]}}) - self.assertNotIn("served_by", relay.reply_facts("/chat/completions", stream, True)) + body = dict(RESPONSES, openrouter_metadata=routing_facts("Prov B", "Prov A")) + self.assertEqual(relay.reply_facts("/responses", json.dumps(body).encode(), False)["served_by"], "Prov B") + + def test_facts_that_mark_no_single_named_endpoint_take_no_earlier_name(self): + both = routing_facts("Prov A", "Prov X") + for e in both["endpoints"]["available"]: + e["selected"] = True + for meta in (routing_facts(None, "Prov A", "Prov X"), routing_facts(7, "Prov A"), both): + with self.subTest(meta=meta): + body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": meta} + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) + stream = sse({"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=meta)}) + self.assertNotIn("served_by", relay.reply_facts("/responses", stream, True)) + + def test_a_pinned_responses_call_is_recorded_with_its_provider_and_the_pin_holds(self): + """A Responses reply names its provider in the routing facts alone: the relay asks for them on a pinned + request to OpenRouter, records the endpoint they mark, and the pin reads held, streamed or not. Without + them the reply names no provider and the pin reads unknown.""" + up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=up.serve_forever, daemon=True).start() + self.addCleanup(up.server_close) + self.addCleanup(up.shutdown) + Upstream.seen, Upstream.next, Upstream.always = [], None, None + base, pin = f"http://127.0.0.1:{up.server_address[1]}/v1", {"m-run": ["prov-c"]} + for facts, who, held in ((True, "Prov C", True), (False, None, "unknown")): + r = relay.Relay(base, "or-test", os.path.join(self.tmp, f"rec-{facts}"), model="m-run", serving=pin, facts=facts).start() + self.addCleanup(r.stop) + for stream in (False, True): + with self.subTest(facts=facts, stream=stream): + post(r.url("u0") + "/responses", body_for("/responses", stream)) + self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled" if facts else None) + self.assertEqual(json.loads(Upstream.seen[-1][2])["provider"], {"order": ["prov-c"], "allow_fallbacks": False}) + self.assertEqual(r.calls("u0")[-1]["reply"].get("served_by"), who) + self.assertEqual(job.served(r.calls("u0"), pin)["held"], {"m-run": held}) def test_an_error_keeps_its_stated_fields_alone(self): nested = relay.reply_facts("/chat/completions", json.dumps({"error": {"api_key": ["review-only-credential"]}}).encode(), False) From f092fa3bdd08ce08707293551de4de9c98be0c53 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:47:31 +0100 Subject: [PATCH 14/24] docs(standard): say a Responses request asks for the routing facts, a pin no provider serves gets 404, and why held may read unknown --- docs/cli/run.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index ee86bf4..4db8efd 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -78,12 +78,12 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | --- | --- | | A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | | Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | -| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`), where its reply names the provider that served it. A model with no pin is sent as before | +| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`): a Responses reply names the provider that served it there alone. A Messages request is sent without: its reply names its provider. A model with no pin is sent as before | | Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | -| No provider can serve it | OpenRouter answers with an error and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider of the reply's end (an earlier, failed attempt's never), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | -| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A provider whose name is not its slug once lowercased reads `unknown` though it served the call: `DeepInfra` is `deepinfra`, and `Google` is not `google-vertex` | +| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name, and one that is not its slug once lowercased reads `unknown` though it served the call: `DeepInfra` is `deepinfra`, but `Google` is not `google-vertex`, nor `SambaNova` `sambanova-turbo` | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | | The builder | *Model* step, **Served by** under each model field, on OpenRouter alone: the recipe's `served_by`, by field (`model`, `simulator`, `judge`), compiled into `--served-by` for each job that calls that model. Its list of a model's providers comes from OpenRouter (`/models/{maker}/{model}/endpoints`), asked as the model list is; when it cannot be had, a slug typed by hand is taken | From fe81ec1d5f974118f4407f54ca1c418c98b03d03 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:48:05 +0100 Subject: [PATCH 15/24] test(standard): check that routing facts marking no single endpoint take no name a chunk gave before --- app/tests/test_standard_served_by.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index 8aa3690..c160df5 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -636,6 +636,9 @@ def test_facts_that_mark_no_single_named_endpoint_take_no_earlier_name(self): stream = sse({"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=meta)}) self.assertNotIn("served_by", relay.reply_facts("/responses", stream, True)) + chunks = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "openrouter_metadata": meta}) + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", chunks, True)) def test_a_pinned_responses_call_is_recorded_with_its_provider_and_the_pin_holds(self): """A Responses reply names its provider in the routing facts alone: the relay asks for them on a pinned From 7b3f8f3c241461b1cbf95288e5195c4fe1663d57 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 16:51:20 +0100 Subject: [PATCH 16/24] refactor(builder): drop a comment written twice above the model field's change --- app/static/builder/command-builder.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 37a3016..949d79c 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -1019,7 +1019,6 @@ else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} // a provider added to a Served by, after the ones already there; the field shows the list, in its order else if(d.servedAdd!==undefined){const rec=r(),role=d.servedAdd,now=(rec.served_by||{})[role]||[];if(el.value&&!now.includes(el.value))C.setServed(rec,role,[...now,el.value]);el.value='';const f=root.querySelector(`[data-served-by="${role}"]`);if(f)f.value=((r().served_by||{})[role]||[]).join(', ');refreshPlan();} - // a model chosen: its Served by lists that model's providers // a model chosen: its Served by lists that model's providers; the keyboard stays on the field the new form holds else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan(); requestAnimationFrame(()=>{const f=root.querySelector(`[data-field="${d.field}"]`);if(f&&document.activeElement!==f)f.focus();});} From d6f6514650fdf0570720df767d835bc2fe2e8b7a Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 17:03:25 +0100 Subject: [PATCH 17/24] docs(standard): say in sentences of their own why a Responses request asks for the routing facts and where held reads unknown --- docs/cli/run.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index 4db8efd..3f82edc 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -78,12 +78,12 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | --- | --- | | A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | | Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | -| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`): a Responses reply names the provider that served it there alone. A Messages request is sent without: its reply names its provider. A model with no pin is sent as before | +| What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`). A Responses reply names the provider that served it there alone. A Messages request is sent without the header, since its reply names its provider. A model with no pin is sent as before | | Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | | No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does | | The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | -| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name, and one that is not its slug once lowercased reads `unknown` though it served the call: `DeepInfra` is `deepinfra`, but `Google` is not `google-vertex`, nor `SambaNova` `sambanova-turbo` | +| What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name. Where that name is not the slug once lowercased, `held` reads `unknown` though the provider served the call (`Google` for `google-vertex`, `SambaNova` for `sambanova-turbo`) | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | | The builder | *Model* step, **Served by** under each model field, on OpenRouter alone: the recipe's `served_by`, by field (`model`, `simulator`, `judge`), compiled into `--served-by` for each job that calls that model. Its list of a model's providers comes from OpenRouter (`/models/{maker}/{model}/endpoints`), asked as the model list is; when it cannot be had, a slug typed by hand is taken | From 7cde14df1eeb9629d8be96e069012899d5a00bd0 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 17:47:16 +0100 Subject: [PATCH 18/24] test(standard): keep a unit list and a serving pin together in a run's record A run given --units and --served-by from the command line records both in config.json, with and without --from-run. --- app/tests/test_standard_units.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index bc642bf..7d7d77a 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -339,6 +339,23 @@ def test_the_command_runs_the_listed_units_with_and_without_from_run(self): self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000", "slow__u0002"]) self.assertEqual(read(os.path.join(job, "config.json"))["standard"]["units_file"]["units"], ["slow", "good"]) + def test_the_command_records_both_the_units_file_and_the_serving_pin(self): + path = self.file("good\n") + self.b["score"] = dict(self.b["score"], judge={"model": "m-judge", "says": "a judge"}) + # The fixture scores locally; no relay listener is needed. + for extra in ([], ["--from-run"]): + with self.subTest(extra=extra), mock.patch.object(run.relay.Relay, "start", lambda r: r), \ + mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9"), \ + mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"}): + code, out = self.cli("--units", path, "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", "--name", "pinned" + "".join(extra), *extra) + self.assertEqual(code, 0, out) + job = out.strip().splitlines()[-1] + standard = read(os.path.join(job, "config.json"))["standard"] + self.assertEqual(standard["units_file"], run.units_file(path)) + self.assertEqual(standard["served_by"], {"m-judge": ["prov-a"]}) + self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000"]) + def test_a_list_the_run_cannot_take_is_refused_with_exit_2(self): for text, said in (("good\ngood\n", "more than once"), ("# none\n", "lists no unit"), ("good\nnope\n", "nope")): with self.subTest(text=text): From a7317dac15b802aefb09295b62e1db50b900dc8f Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 17:47:16 +0100 Subject: [PATCH 19/24] docs(standard): describe a job's unit list and serving pin together in the field guide The field guide names both config fields, the routing fields of a call, the serving counters and held, and says that merge refuses different unit lists always and different pins unless forced. --- docs/field-guide.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/field-guide.md b/docs/field-guide.md index 8fbffd3..0f243cd 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -169,6 +169,18 @@ How the agent worked, not whether it won. Computed over layer 3. | `exception_type` | What ended the trial, if anything. It decides the infra / agent split. | measured | any | | `n_rate_limited_retries` | 429s counted in `agent/gateway.log`: on Harbor, the ones the agent's SDK retried and survived; on a standard run, every model call the relay saw refused with 429. No log reads *not recorded*, never 0. | inferred | any | +## Standard run records + +These fields are stored in the standard's job and trial files ([Run a benchmark](cli/run.md)). + +| Field | What it is | Source | Works on | +| --- | --- | --- | --- | +| `standard.units_file` | In `config.json`: the units file's name, sha256 and ordered ids, before sharding and the limit. Units run in dataset order; merging requires the same listed units, even with `--force`, and keeps the first shard's file record. | measured | standard | +| `standard.served_by` | In the same config: the allowed OpenRouter provider slugs by model, in the order tried. A job can record both its units file and its serving pins; merging different pins requires `--force`. | measured | standard | +| `served / standard.served` | In the job's `result.json` and each trial's `result.json`, respectively: answered calls by provider (`calls`), calls naming none (`unnamed`), and each pinned model's `held`. Counts cover every relay route; the job also includes the benchmark's own relay. Tokens and cost retain their existing scope. | measured | standard | +| `held` | `true` when every answered call names an allowed provider, `unknown` for an absent or different name, and `null` for no answered calls. Names are lowercased with spaces as dashes and compared with the slug before `/`; endpoint variants cannot be verified. Merging different pins marks the affected models `unknown`. | inferred | standard | +| `request.routing / request.routing_asked / reply.served_by / reply.end / reply.error` | In `agent/calls.jsonl`: routing sent, the caller's routing when replaced, the answering provider, the reply's ending and any error inside a 200 response. These credential-cleaned fields leave the call's status and the unit's verdict unchanged. | measured | standard | + ## Trajectory metrics, inside a single run {#trajectory} | Field | What it is | Source | Works on | From a80ca4e6bc269d9b85ebf8ebaf2be4664120a899 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 18:30:14 +0100 Subject: [PATCH 20/24] test(standard): run a serving pin under a machine's constraints Seven cases where the two meet: both records from the command line, with and without --from-run; the order of the refusals; a scope's enforcement with a valid pin; a container relay's caps beside its pins; each target's compiled command; the builder's checks; and shards whose machine limits differ. --- .../test_standard_served_by_constraints.py | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 app/tests/test_standard_served_by_constraints.py diff --git a/app/tests/test_standard_served_by_constraints.py b/app/tests/test_standard_served_by_constraints.py new file mode 100644 index 0000000..b606c49 --- /dev/null +++ b/app/tests/test_standard_served_by_constraints.py @@ -0,0 +1,172 @@ +"""Provider pins and machine constraints on the same run and recipe. + +These intersections need both features: neither side's tests exercise them +together. The fixture driver scores locally; no endpoint or engine is used. +""" +from __future__ import annotations + +import io +import json +import os +from pathlib import Path +import shlex +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from nullius import constraints, progress +from nullius.standard import cli, container, definitions, install, job, merge, recipe, run +from tests import test_standard_recipe_served_by as recipes + + +class Together(unittest.TestCase): + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(mock.patch.object(install, "HOME", str(self.tmp / "install"))) + self.addCleanup(container.cap, {}) + self.b = recipes.judged() + self.a = definitions.get("agent", "fake", recipes.ROOT) + self.pin = {"m-judge": ["prov-a"]} + + def make(self, **kw): + values = dict(model="m", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, + judge_model="m-judge", served_by=self.pin, unsandboxed=True, + jobs_root=str(self.tmp), say=lambda _: None) + values.update(kw) + return run.Run(self.b, self.a, **values) + + def test_the_cli_records_units_constraints_and_pins_together(self): + units = self.tmp / "units.txt" + units.write_text("good\n") + self.enterContext(mock.patch.object(cli, "_bench", return_value=self.b)) + self.enterContext(mock.patch.object(cli, "_exit_on_term")) + self.enterContext(mock.patch.object(cli.settings, "load")) + self.enterContext(mock.patch.object(progress, "ECHO", None)) + self.enterContext(mock.patch.object(run.relay.Relay, "start", lambda r: r)) + self.enterContext(mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9")) + self.enterContext(mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"})) + seen = [] + proc = run.jsonl.Proc + + def driver(cmd, cwd, env, log, **kw): + seen.append(env) + return proc(cmd, cwd, env, log, **kw) + + self.enterContext(mock.patch.object(run.jsonl, "Proc", driver)) + for i, extra in enumerate(([], ["--from-run"])): + out = io.StringIO() + with self.subTest(extra=extra), redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--jobs", str(self.tmp), "--name", f"job-{i}", + "--units", str(units), "-j", "4", "--constraint", "concurrency=1", + "--constraint", "gpu_ids=", "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", *extra]) + self.assertEqual(code, 0, out.getvalue()) + path = Path(out.getvalue().strip().splitlines()[-1]) + cfg = json.loads((path / "config.json").read_text()) + self.assertEqual(cfg["standard"]["units_file"], run.units_file(str(units))) + self.assertEqual(cfg["standard"]["constraints"], {"concurrency": 1, "gpu_ids": []}) + self.assertEqual(cfg["standard"]["served_by"], self.pin) + self.assertEqual(cfg["n_concurrent_trials"], 1) + self.assertEqual(sorted(p.name for p in path.iterdir() if "__u" in p.name), ["good__u0000"]) + self.assertEqual(json.loads((path / "result.json").read_text())["served"]["held"], {"m-judge": None}) + self.assertTrue(seen) + self.assertTrue(all(env["CUDA_VISIBLE_DEVICES"] == "" for env in seen)) + + def test_constraint_and_host_refusals_precede_pin_and_endpoint_checks(self): + with mock.patch.object(run, "provider_endpoint") as endpoint: + with self.assertRaisesRegex(run.Refused, "constraints:.*at least 1"): + self.make(constraints={"concurrency": 0}, served_by={"stray": ["prov-a"]}) + self.b = dict(self.b, host={"gpu": {"check": ["nvidia-smi"]}}, + options={"compute": {"default": "gpu", "values": ["gpu"], "needs": {"gpu": {"host": ["gpu"]}}}}) + with self.assertRaisesRegex(run.Refused, "allows Nullius no GPU"): + self.make(constraints={"gpu_ids": []}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + with self.assertRaisesRegex(run.Refused, "--served-by.*does not call"): + self.make(constraints={"gpu_ids": [0]}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + endpoint.assert_not_called() + out = io.StringIO() + with mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_exit_on_term"), \ + mock.patch.object(cli.settings, "load"), mock.patch.object(run, "run_and_report") as start, redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--constraint", "concurrency=0", "--served-by", "bad slug"]) + self.assertEqual(code, 2) + self.assertIn("refused: constraints:", out.getvalue()) + start.assert_not_called() + + def test_a_valid_pin_still_requires_enforced_machine_caps(self): + r = self.make(constraints={"cpus": 1}) + with mock.patch.object(constraints, "scope_check", return_value={"why": "unavailable", "enforced": [], "refused": {}}), \ + mock.patch.object(install, "packages") as setup, self.assertRaisesRegex(run.Refused, "systemd does not enforce"): + r.ready() + setup.assert_not_called() + + def test_the_container_relay_takes_both_caps_and_pins(self): + self.enterContext(mock.patch.object(container, "_security", "")) + with mock.patch.object(container, "unusable", return_value=None): + r = self.make(backend="container", constraints={"cpus": 1, "memory_mb": 512}) + argv = container.relay_argv("img", name="relay", job="job", app="/app", record="/rec", key_file="/key", + base="https://openrouter.ai/api/v1", model="m", routes={"u0-judge": "m-judge"}, serving=r.serving) + self.assertEqual(argv[argv.index("--cpus") + 1], "1") + self.assertEqual(argv[argv.index("--memory") + 1], "512m") + self.assertEqual(argv[argv.index("--served-by") + 1], "m-judge=prov-a") + with mock.patch.object(r, "scope_ready", return_value={}), \ + mock.patch.object(container, "base_tag", return_value="img"), \ + mock.patch.object(container, "present", return_value=True), \ + mock.patch.object(container, "base_image", return_value="img"), \ + mock.patch.object(container, "unenforced", return_value={"cpus": "max 100000"}), \ + self.assertRaisesRegex(run.Refused, "engine does not apply"): + r.ready() + + def test_each_compiled_shard_has_its_constraints_and_each_called_models_pin(self): + given = {"local": {"concurrency": 1}, "127.0.0.1": {"cpus": 2, "gpu_ids": []}} + with mock.patch.dict(os.environ, {constraints.SETTING: json.dumps(given)}): + out = recipe.compile(recipes.a_recipe(where=list(given), others=["fake", "nop"], + model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), recipes.defs(self.b)) + self.assertEqual(out["errors"], []) + self.assertEqual(out["served_models"]["model"], "openai/gpt-4o-mini") + for part in out["shards"]: + for line in part["command"].splitlines(): + words = shlex.split(line) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--constraint"], + constraints.cli_words(given[part["where"]])[1::2]) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--served-by"], + ["openai/gpt-4o-mini=prov-a"]) + self.assertLess(words.index("--constraint"), words.index("--served-by")) + + def test_builder_keeps_both_validation_errors_in_main_first_order(self): + with mock.patch.dict(os.environ, {constraints.SETTING: '{"local": {"cpus": 0}}'}): + out = recipe.compile(recipes.a_recipe(model="openai/gpt-4o-mini", + served_by={"model": ["prov-a"], "judge": ["prov-b"]}), recipes.defs(self.b)) + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"], recipe.STEP["model"]]) + self.assertIn("HOST_CONSTRAINTS", out["errors"][0]["message"]) + self.assertIn("pins openai/gpt-4o-mini twice", out["errors"][1]["message"]) + self.assertEqual((out["script"], out["parts"]), ("", [])) + + def test_merge_allows_machine_limits_to_differ_but_still_checks_pins(self): + paths = [] + for i in (1, 2): + path = self.tmp / f"shard-{i}" + path.mkdir() + job.write_config(str(path), job=path.name, bench=self.b, other=self.a, model="m", provider="openrouter", + dataset="all", units=[], concurrency=i, granted=True, launch="launch", + shard={"index": i, "of": 2, "units_total": 0, "planned": []}, + constraints={"concurrency": i}, served_by=self.pin) + (path / "result.json").write_text(json.dumps({"served": {"calls": {"Prov A": 1}, "unnamed": 0, + "held": {"m-judge": True}}})) + paths.append(str(path)) + self.assertEqual(merge.check(paths)["differ"], []) + merge.merge(paths, str(self.tmp / "joined")) + cfg = json.loads((self.tmp / "joined/config.json").read_text())["standard"] + self.assertEqual((cfg["constraints"], cfg["served_by"]), ({"concurrency": 1}, self.pin)) + self.assertEqual(json.loads((self.tmp / "joined/result.json").read_text())["served"]["held"], {"m-judge": True}) + path = Path(paths[1]) / "config.json" + cfg = json.loads(path.read_text()) + cfg["standard"]["served_by"] = {"m-judge": ["prov-b"]} + path.write_text(json.dumps(cfg)) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(paths) + merge.merge(paths, str(self.tmp / "forced"), force=True) + self.assertEqual(json.loads((self.tmp / "forced/result.json").read_text())["served"]["held"], {"m-judge": "unknown"}) + + +if __name__ == "__main__": + unittest.main() From 468406d8e55f7c7e42064475fdb167a00e511007 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Fri, 13 Nov 2026 18:30:14 +0100 Subject: [PATCH 21/24] docs(standard): name a job's constraints beside its unit list and serving pin in the field guide The field guide says that standard.constraints, standard.units_file and standard.served_by may stand together in config.json, and which of them a merged job keeps from its first shard. --- docs/field-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/field-guide.md b/docs/field-guide.md index 0f243cd..f157345 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -176,6 +176,7 @@ These fields are stored in the standard's job and trial files ([Run a benchmark] | Field | What it is | Source | Works on | | --- | --- | --- | --- | | `standard.units_file` | In `config.json`: the units file's name, sha256 and ordered ids, before sharding and the limit. Units run in dataset order; merging requires the same listed units, even with `--force`, and keeps the first shard's file record. | measured | standard | +| `standard.constraints` | In the same config: what the machine allowed the run, when set. These values can coexist with a units file and serving pins. Shards may have different machine constraints; merging keeps the first shard's config values. | measured | standard | | `standard.served_by` | In the same config: the allowed OpenRouter provider slugs by model, in the order tried. A job can record both its units file and its serving pins; merging different pins requires `--force`. | measured | standard | | `served / standard.served` | In the job's `result.json` and each trial's `result.json`, respectively: answered calls by provider (`calls`), calls naming none (`unnamed`), and each pinned model's `held`. Counts cover every relay route; the job also includes the benchmark's own relay. Tokens and cost retain their existing scope. | measured | standard | | `held` | `true` when every answered call names an allowed provider, `unknown` for an absent or different name, and `null` for no answered calls. Names are lowercased with spaces as dashes and compared with the slug before `/`; endpoint variants cannot be verified. Merging different pins marks the affected models `unknown`. | inferred | standard | From 7e92f43fc7cf4a1229c08b949dfc7f62817b93ca Mon Sep 17 00:00:00 2001 From: Oleg <247799477+sirotaoleg@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:10:40 +0000 Subject: [PATCH 22/24] feat(standard): give the standard what computer-use benchmarks need, on top of --served-by: gated tasks, a definition's pins as its serving pins, a pinned route's refusal as the unit's, limits of their own, and prove of a merged control A definition's pin (setups.original.pin, score.judge.pin) joins the run's --served-by pins and wins over --served-by for its model; ApiProviderRouteError reads the pin and reply.served_by. A failed Responses object reads reply.end and reply.error. --- app/nullius/standard/cli.py | 13 +- app/nullius/standard/container.py | 11 +- app/nullius/standard/definitions.py | 75 ++++- app/nullius/standard/fidelity.py | 5 +- app/nullius/standard/job.py | 34 ++- app/nullius/standard/merge.py | 47 ++- app/nullius/standard/prove.py | 204 +++++++++++-- app/nullius/standard/recipe.py | 13 +- app/nullius/standard/relay.py | 234 +++++++++++++-- app/nullius/standard/run.py | 119 +++++++- app/tests/test_dependency_surface.py | 21 ++ app/tests/test_standard_context_window.py | 287 ++++++++++++++++++ app/tests/test_standard_controls.py | 343 ++++++++++++++++++++++ app/tests/test_standard_gated.py | 134 +++++++++ app/tests/test_standard_parts.py | 9 + app/tests/test_standard_relay.py | 174 ++++++++++- app/tests/test_standard_relay_pin.py | 263 +++++++++++++++++ app/tests/test_standard_run.py | 19 ++ app/tests/test_standard_served_by.py | 15 +- app/tests/test_standard_shard.py | 39 ++- app/tests/test_standard_trace.py | 26 ++ app/tests/test_standard_units.py | 2 +- app/tests/test_standard_versions.py | 2 +- docs/cli/read-jobs.md | 2 +- docs/cli/run.md | 6 +- docs/standard.md | 26 +- 26 files changed, 1993 insertions(+), 130 deletions(-) create mode 100644 app/tests/test_standard_context_window.py create mode 100644 app/tests/test_standard_gated.py create mode 100644 app/tests/test_standard_relay_pin.py diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index d8b1cb6..2cd0065 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -9,7 +9,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] - PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] + PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force] PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed @@ -600,11 +600,16 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) + jobs = options(a.job) + for c in jobs: + if c not in definitions.CONTROLS: + sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}") held, spill = licences(b) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, - env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images) + env=run_env(a.provider) if a.from_run else None, licences=held, + bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images) except run.Refused as e: print("refused:", e) return 2 @@ -714,6 +719,10 @@ def main(argv=None): v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)") v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only") v.add_argument("--launch", help="the launch id the control jobs of every shard record") + v.add_argument("--bench-option", action="append", metavar="NAME=VALUE", + help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs") + v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR", + help="hold this job, a control run already over the whole dataset (a merged launch), to its bound instead of running it") v.add_argument("--remove-images", action="store_true", help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)") g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bacddab..9153d21 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + timeout: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -761,6 +762,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] + if timeout: + a += ["--timeout", str(timeout)] return a @@ -1022,10 +1025,11 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, - serving: Optional[Dict[str, List[str]]] = None): + serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one self.serving = dict(serving or {}) # by model, the serving providers that may answer it + self.timeout = timeout self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1051,7 +1055,8 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger, serving=self.serving) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, + timeout=self.timeout) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 125c2b2..0a6e2b1 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -81,7 +81,7 @@ import re from typing import Callable, Dict, List, Optional, Tuple -from nullius.standard import ports +from nullius.standard import ports, relay from nullius import paths from nullius.catalogue import storage from . import names @@ -1035,6 +1035,25 @@ def control(bench: dict, which: str) -> Optional[dict]: "_kind": "control", "_ref": which, "_here": bench.get("_here", "")} +def comparable(orig: dict, model: str) -> bool: + """Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do.""" + c = orig.get("comparable", True) + return model in c if isinstance(c, list) else bool(c) + + +def upstream_name(orig: dict, model: str, provider: str) -> str: + """``model`` by the name the original's upstream knows, when given as ``provider`` lists it.""" + pre = (orig.get("providers") or {}).get(provider, "") + names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []} + return model if model in (orig.get("models") or []) else names.get(model, model) + + +def paper_models(orig: dict) -> List[str]: + """The original's models whose runs compare with the paper's.""" + c = orig.get("comparable", True) + return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c] + + def _leaderboard(d: dict) -> List[str]: """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the upstream's leaderboard fixes besides the ranked model, from its own submission @@ -1066,12 +1085,13 @@ def _leaderboard(d: dict) -> List[str]: def _setups(s) -> List[str]: """``setups``: {original: {summary, models, default, served_as?, providers, - side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s + side_models?, pin?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s ``models`` are the paper's, by the names the upstream's code knows them; ``side_models`` the ones its code asks besides the answerer (an extraction, a summary), sent as asked through served_as and the provider's prefix; ``providers`` the ones that serve them, each with the prefix its names - carry there; ``model`` needs ``original`` (its pipeline) and asks as one + carry there; ``pin`` {openrouter: {model: [provider slug]}}, the only + providers of each; ``model`` needs ``original`` (its pipeline) and asks as one of its models (``runs_as``).""" if s is None: return [] @@ -1090,7 +1110,7 @@ def _setups(s) -> List[str]: bad.append(f"{w}.summary: what runs, in a sentence") allowed = {"summary"} if k == "original": - allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"} + allowed |= {"models", "default", "served_as", "providers", "side_models", "pin", "units", "turn_seconds", "paper", "comparable", "reference_agent"} if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()): bad.append(f"{w}.reference_agent: the upstream agent's display name") models = x.get("models") @@ -1114,6 +1134,8 @@ def _setups(s) -> List[str]: pv = x.get("providers") if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()): bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}") + if "pin" in x: + bad += _pin(x["pin"], f"{w}.pin", pv if isinstance(pv, dict) else {}, set(models + side)) u = x.get("units") if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))): bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is") @@ -1122,8 +1144,11 @@ def _setups(s) -> List[str]: bad.append(f"{w}.turn_seconds: a whole number of seconds") if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()): bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase") - if "comparable" in x and not isinstance(x["comparable"], bool): - bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only") + c = x.get("comparable", True) + if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c) + or x.get("default") not in c): + bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; " + "or those of its models whose runs compare with the paper's, its default among them") else: allowed |= {"runs_as"} orig = s.get("original") if isinstance(s.get("original"), dict) else None @@ -1139,6 +1164,28 @@ def _setups(s) -> List[str]: return bad +PIN_PROVIDERS = ("openrouter",) + + +def _pin(pin, w: str, providers: dict, models: Optional[set] = None) -> List[str]: + """``pin``: {provider: {model: [slug]}} for a setup (``models`` given), {provider: [slug]} for a judge.""" + if not isinstance(pin, dict) or not pin: + return [f"{w}: {{openrouter: ...}}, the providers that alone serve it"] + bad = [] + for p, v in pin.items(): + if p not in PIN_PROVIDERS or (providers and p not in providers): + bad.append(f"{w}: a pin is OpenRouter's provider routing, on a provider the benchmark runs on ({p!r})") + continue + pins = v if models is not None else {"_": v} + if not isinstance(pins, dict) or not pins: + bad.append(f"{w}.{p}: {{model: [provider slug, ...]}}") + continue + if models is not None and not all(m in models for m in pins): + bad.append(f"{w}.{p}: pins only its models and side_models") + bad += [f"{w}.{p}: " + b.removeprefix("_: ") for b in relay.serving_problems(pins)] + return bad + + # Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder). SETTINGS_FILE = "upstream-settings.json" _DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$") @@ -1204,7 +1251,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple return None, f"{bench['label']} has no {which} setup" orig, x = s["original"], s[which] model = (model or "").strip() - side = [] + side, pins = [], {} if which == "original" and "models" not in orig: # any model, sent as the provider lists it if not model: return None, f"the original {bench['label']} needs a model: its leaderboard ranks models" @@ -1215,7 +1262,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple model = model or orig["default"] if model not in orig["models"]: return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): " - f"{model} is not one - choose the model setup for another") + f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else "")) if provider and provider not in orig["providers"]: return None, (f"the paper's models are not served by {provider}: the original runs on " + " or ".join(orig["providers"])) @@ -1225,13 +1272,15 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple sent = served(model) # the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent] + pins = {served(m): list(only) for m, only in ((orig.get("pin") or {}).get(provider) or {}).items() + if m == model or m in (orig.get("side_models") or [])} else: if not model: return None, f"the model setup needs a model: the one to put in {orig['default']}'s place" asked, sent = x["runs_as"], model units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"} - return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", - "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True), + return {"model": {"side_models": side}, **({"pins": pins} if pins else {}), "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", + "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model), "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which, "_here": bench.get("_here", "")}, "" @@ -1313,7 +1362,7 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") @@ -1321,6 +1370,8 @@ def problems(d: dict, kind: str) -> List[str]: if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " "in a sentence - the driver is then sent attempt with setup, turn and answer") + if "gated" in d and not isinstance(d["gated"], bool): + bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept") s = d.get("score") or {} if s.get("kind") not in SCORES or not s.get("says"): bad.append(f"score: {{kind: one of {SCORES}, says}}") @@ -1332,6 +1383,8 @@ def problems(d: dict, kind: str) -> List[str]: j = s.get("judge") if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")): bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}") + elif j is not None and "pin" in j: + bad += _pin(j["pin"], "score.judge.pin", {}) sim = d.get("simulator") if sim is not None: if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"): diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index bc60de2..cd79086 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,8 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() + key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True, + timeout=r.call_seconds).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} @@ -230,7 +231,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k except (run._Ended, jsonl.Failed) as e: r.say(f" {tag} {unit}: {e}") try: - score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600) + score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds) except jsonl.Failed: score = {} if os.path.isdir(os.path.join(base, "agent")): diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 6234e4e..a4cd73c 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -57,9 +57,9 @@ def _text(content) -> str: def _usage_total(calls: List[dict]) -> dict: - ok = [c for c in calls if c.get("status") == 200] + ok = [c for c in calls if c.get("status") == 200 or c.get("reply")] u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok) - costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))] + costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float)) and c.get("cost_from") != "failed"] return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"), "cost": sum(costs) if costs else None, "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None), @@ -169,6 +169,9 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili reply = c.get("reply") or {} st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"), "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"} + if c.get("status") == 200 and reply.get("tool_calls"): + st["tool_calls"] = [{"tool_call_id": f"call-{len(steps) + 1}-{k + 1}", "function_name": t.get("name") or "", + "arguments": _arguments(t.get("arguments"))} for k, t in enumerate(reply["tool_calls"])] if reply.get("usage"): st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"], "cached_tokens": reply["usage"]["cached"]} @@ -183,6 +186,17 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}} +def _arguments(said) -> dict: + """A tool call's arguments as a dict; text that is no JSON object goes under ``raw``.""" + try: + got = json.loads(said) if isinstance(said, str) else said + except ValueError: + got = None + if said is None: + return {} + return got if isinstance(got, dict) else {"raw": said if isinstance(said, str) else json.dumps(said)} + + def _control(other: dict) -> dict: """A control's run says so, and who wrote the control: the runs pages read a standard job named ``nop`` or ``oracle`` as a control only by this.""" @@ -317,7 +331,7 @@ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tu if at is not None: turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True, content={"input": st.get("message")})) - elif agent or st.get("tool_calls"): + elif agent or (st.get("tool_calls") and not st.get("model_name")): spans.extend(_step_spans(i + 1, st, source)) if not spans and not turns: return None @@ -497,6 +511,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st # benchmark a score and no pass or fail - the trial reads as Scored rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value} res["verifier_result"] = {"rewards": rewards} + if score.get("stopped"): + res["verifier_result"]["stopped"] = str(score["stopped"]) parts = score.get("parts") if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0: # a score over several parts (a conversation's questions): the job's mean weighs it by them @@ -515,9 +531,15 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) if trajectory is not None: _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1)) - timeline = trace_from_run(trajectory, calls, other, produce) - if timeline is not None: - _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + try: + timeline = trace_from_run(trajectory, calls, other, produce) + if timeline is not None: + _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + except Exception as e: + try: + _write(os.path.join(trial_dir, "agent", "trace.error.txt"), type(e).__name__ + "\n") + except OSError: + pass for name, text in (extra_logs or {}).items(): _write(os.path.join(trial_dir, "agent", name), text) _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1)) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 3f0a46a..955df60 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -19,7 +19,11 @@ comments, line endings and sha256 may differ from shard to shard, and the merged job records the first shard's list. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: -the shards absent, and the units a shard planned and has no trial for. +the shards absent, and the units a shard planned and has no trial for. A +trial whose record names another job, benchmark, version, control or agent +than its shard, or a job itself merged despite differences, is refused +unless ``force``. Shards run with different benchmark options keep them each +in ``merged_from``. Files are hard-linked into the merged job where the file system allows (a copy where it does not), so a merge of a thousand trials costs no second @@ -45,6 +49,11 @@ ("number of shards", ("standard", "shard", "of"))) +LISTED = 1000 +OWN = (("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), + ("control", ("standard", "control", "name")), ("agent or capability", ("standard", "plugged"))) + + class Refused(Exception): """The jobs cannot be merged as asked; the message says why.""" @@ -155,12 +164,30 @@ def check(job_dirs: List[str], force: bool = False) -> dict: # every trial by its unit: two of one unit are never merged, force or not by_unit: Dict[str, List[tuple]] = {} names: Dict[str, List[str]] = {} + foreign = [] for s in shards: s["trials"] = _trials(s["dir"]) for t in s["trials"]: - unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t + res = _read(os.path.join(s["dir"], t, "result.json")) + unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) + wrong = [label for label, path in OWN if _at(res, path) != _at(s["config"], path)] + if _at(res, ("standard", "job")) != (s["config"].get("job_name") or s["name"]): + wrong.append("job") + if wrong: + foreign.append(f"{s['name']}/{t} names another {', '.join(wrong)} than its job") + inherited = [f"{s['name']}: {w}" for s in shards for w in _at(s["config"], ("standard", "merged_despite")) or []] + if inherited: + if not force: + raise Refused("a job given was itself merged despite differences - " + "; ".join(inherited[:5]) + + " (--force merges it anyway, and the merge says so)") + differ += inherited + if foreign: + shown = f"{len(foreign)} trial(s) not of the job they are in: " + "; ".join(foreign[:5]) + (" ..." if len(foreign) > 5 else "") + if not force: + raise Refused(shown + " (--force merges them anyway, and the merge says so)") + differ.append(shown) twice = {u: w for u, w in by_unit.items() if len(w) > 1} if twice: shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10]) @@ -175,9 +202,10 @@ def check(job_dirs: List[str], force: bool = False) -> dict: if len(of) == 1: n = of.pop() have = sorted(s["shard"].get("index") for s in shards if s["shard"]) - absent = [i for i in range(1, (n or 0) + 1) if i not in have] - if absent: - notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") + if isinstance(n, int) and n <= LISTED: + absent = [i for i in range(1, n + 1) if i not in have] + if absent: + notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") dup = sorted({i for i in have if have.count(i) > 1}) if dup: notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)") @@ -271,8 +299,15 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: cfg["job_name"] = name std = cfg.setdefault("standard", {}) std.pop("shard", None) + own = len({json.dumps(_at(s["config"], ("standard", "benchmark_options")), sort_keys=True) for s in shards}) > 1 + if own: + std.pop("benchmark_options", None) std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]), - **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards] + **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {}), + **({"benchmark_options": _at(s["config"], ("standard", "benchmark_options")) or {}} if own else {}), + **({"merged_from": _at(s["config"], ("standard", "merged_from"))} + if _at(s["config"], ("standard", "merged_from")) else {})} + for s in shards] if got["units_total"] is not None: std["units_total"] = got["units_total"] if got["differ"]: diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318..8b39142 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -30,6 +30,8 @@ pages pair them as a launch's controls); the units and leak checks talk to the driver directly and leave nothing behind. A control calls no model, so it needs no key - except a judged benchmark's, whose judge still grades. +``--bench-option`` reaches the checks' driver and each control's run; +``--job nop=DIR`` holds a merged launch of a control to its bound. A check that passes is recorded with its date, its counts and its job; one that fails is taken back, as ``conform`` takes back a port - what it reached kept @@ -41,6 +43,8 @@ """ from __future__ import annotations +import json +import math import os import re import shutil @@ -48,8 +52,7 @@ import time from typing import Callable, Dict, List, Optional -from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, run, shard as shards +from nullius.standard import definitions, install, job, jsonl, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -66,7 +69,8 @@ class _Driver: """The benchmark's driver, started as a run starts it, for the checks that talk to it directly (units, leak).""" - def __init__(self, bench: dict, ds: dict, work: str, licences=None): + def __init__(self, bench: dict, ds: dict, work: str, licences=None, options: Optional[dict] = None): + self.options = options if options is not None else options_for(bench, ds) self.up = install.upstream(bench) self.data = install.dataset(bench, ds, self.up) install.driver_packages(bench) # its driver.install, once: start() then finds it @@ -81,10 +85,15 @@ def start(self, n: int) -> jsonl.Proc: env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), - **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults + **run.option_env(self.options)}) return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() +def options_for(bench: dict, ds: dict, given: Optional[dict] = None, which=shutil.which) -> dict: + """The benchmark's options on ``ds`` for a control's run, at ``given`` or their defaults.""" + return run.resolve_bench_options(bench, ds["id"], given, "yours", which)[0] + + def units_check(driver: _Driver, ds: dict) -> dict: """Two drivers, one ``units`` call each: the same ids, unique, plain.""" got = [] @@ -209,44 +218,164 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str, provider: str, model: str, judge_model: Optional[str], env, concurrency: int, say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None, - licences=None, remove_images: bool = False) -> dict: + licences=None, options: Optional[dict] = None, job_dir: Optional[str] = None, + ids: Optional[List[str]] = None, remove_images: bool = False) -> dict: """Run one control over the dataset (or its first ``limit`` units, of - ``shard`` when given) and hold it to its bound: the pass rate, or the mean score.""" + ``shard`` when given) and hold it to its bound: the pass rate, or the mean score. + ``job_dir``: a control run already, held to the bound instead; it must cover ``ids``.""" other = definitions.control(bench, which) - job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, - jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), - env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images) - j = parse.parse_job(job_dir) - n = len(j.trials) - errored = sum(1 for t in j.trials if t.exception_type) + problems = [] + if job_dir: + job_dir = os.path.abspath(job_dir.rstrip("/")) + problems += job_problems(bench, ds, which, job_dir, ids or []) + cfg = _config(job_dir) + else: + job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, + jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), + say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, + licences=licences, bench_options=options, + remove_images=remove_images) + cfg = {} + field = FIELD[bench["score"]["kind"]] + results = [res for _, res in trials(job_dir)] + n = len(results) + errored = sum(1 for res in results if res.get("exception_info")) + values = [verdict(bench, res) for res in results if not res.get("exception_info")] + unscored = sum(1 for v in values if v is None) + values = [v for v in values if v is not None] + top = sum(1 for v in values if v >= 1.0) if bench["score"]["kind"] == "pass": - top = sum(1 for t in j.trials if t.reward == 1.0) value = top / n if n else 0.0 else: - scores = [t.score for t in j.trials if t.score is not None] - top = sum(1 for s in scores if s >= 1.0) - value = sum(scores) / len(scores) if scores else 0.0 + value = sum(values) / len(values) if values else 0.0 (bound, expected), = other["expect"].items() within = value <= expected if bound == "at_most" else value >= expected - problems = [] if not n: problems.append("no unit ran") if errored: problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials") + if unscored: + problems.append(f"{unscored} unit(s) ended with no {field} from 0 to 1 and no error: read the job's trials") if n and not within: problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}") rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected}, "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems} - if judge_model or (bench["score"].get("judge") or {}).get("model"): - rec["judge"] = judge_model or bench["score"]["judge"]["model"] + ran_with = _options_of(cfg) if cfg else (options or {}) + if ran_with: + rec["options"] = ran_with + judge = (cfg.get("standard") or {}).get("judge") if cfg else judge_model or (bench["score"].get("judge") or {}).get("model") + if judge: + rec["judge"] = judge return rec +FIELD = {"pass": "reward", "score": "score"} +UNITS_FAILED = "the units check failed" + + +def trials(job_dir: str) -> List[tuple]: + """(trial, its result.json) for each trial folder of a job; an unreadable record is {}.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if name.startswith(".") or not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + res = json.load(f) + except (OSError, ValueError): + res = {} + out.append((name, res if isinstance(res, dict) else {})) + return out + + +def verdict(bench: dict, res: dict) -> Optional[float]: + """The trial's value in the field its benchmark's score kind names, when it is a finite number from 0 to 1.""" + v = (((res.get("verifier_result") or {}).get("rewards")) or {}).get(FIELD[bench["score"]["kind"]]) + if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0.0 <= v <= 1.0: + return None + return float(v) + + +def _options_of(cfg: dict): + """The benchmark options a job ran with: one set, or each distinct set of its merged shards.""" + std = cfg.get("standard") or {} + if std.get("benchmark_options"): + return std["benchmark_options"] + sets, todo = [], list(std.get("merged_from") or []) + while todo: + m = todo.pop() + if m.get("benchmark_options") or not m.get("merged_from"): + sets.append(m.get("benchmark_options") or {}) + else: + todo += m.get("merged_from") or [] + distinct = {json.dumps(x, sort_keys=True): x for x in sets} + return [distinct[k] for k in sorted(distinct)] if any(sets) else [] + + +def _config(job_dir: str) -> dict: + try: + with open(os.path.join(job_dir, "config.json"), encoding="utf-8") as f: + got = json.load(f) + return got if isinstance(got, dict) else {} + except (OSError, ValueError): + return {} + + +def job_problems(bench: dict, ds: dict, which: str, job_dir: str, ids: List[str]) -> List[str]: + """Why ``job_dir`` is not one run of ``which`` over each unit of ``ids`` once.""" + cfg = _config(job_dir) + std = cfg.get("standard") or {} + if cfg.get("runner") != "standard": + return [f"{job_dir}: not a job of the Nullius standard"] + want = {"benchmark": bench["key"], "benchmark version": job._version(bench).get("benchmark_version"), + "control": which, "dataset": f"{bench['key']}/{ds['id']}"} + have = {"benchmark": std.get("benchmark"), "benchmark version": std.get("benchmark_version"), + "control": (std.get("control") or {}).get("name"), + "dataset": ((cfg.get("datasets") or [{}])[0] or {}).get("name")} + bad = [f"its {k} is {have[k]!r}, not {want[k]!r}" for k in want if have[k] != want[k]] + if std.get("merged_despite"): + bad.append("merged despite differences: " + "; ".join(std["merged_despite"])) + if (std.get("repeat") or 1) > 1: + bad.append(f"each unit ran {std['repeat']} times") + ran: Dict[str, int] = {} + shard_jobs = {m.get("job") for m in std.get("merged_from") or []} + alien: Dict[str, List[str]] = {} + for trial, res in trials(job_dir): + tstd = res.get("standard") if isinstance(res.get("standard"), dict) else {} + own = {"benchmark": tstd.get("benchmark"), "benchmark version": tstd.get("benchmark_version"), + "control": (tstd.get("control") or {}).get("name"), + "who ran it": (tstd.get("plugged"), (res.get("agent_info") or {}).get("name")), + "job": tstd.get("job")} + expect = {"benchmark": want["benchmark"], "benchmark version": want["benchmark version"], "control": which, + "who ran it": (which, which), "job": cfg.get("job_name")} + wrong = [k for k in expect if own[k] != expect[k]] + if shard_jobs and tstd.get("shard_job") not in shard_jobs: + wrong.append("shard") + for k in wrong: + alien.setdefault(k, []).append(trial) + u = str(tstd.get("unit") or trial) + ran[u] = ran.get(u, 0) + 1 + for k, names in alien.items(): + bad.append(f"{len(names)} trial(s) whose own record names another {k} than the job: {', '.join(names[:3])}") + known = set(ids) + missing = [u for u in ids if u not in ran] + extra = sorted(u for u in ran if u not in known) + twice = sorted(u for u, k in ran.items() if k > 1) + if missing: + bad.append(f"{len(missing)} of the dataset's {len(ids)} units have no trial: {', '.join(missing[:5])}" + (" ..." if len(missing) > 5 else "")) + if extra: + bad.append(f"{len(extra)} trial(s) of units the dataset does not have: {', '.join(extra[:5])}") + if twice: + bad.append(f"{len(twice)} unit(s) with more than one trial: {', '.join(twice[:5])}") + return [f"{os.path.basename(job_dir)}: {b}" for b in bad] + + def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs", provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None, env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print, shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None, + bench_options: Optional[dict] = None, jobs: Optional[Dict[str, str]] = None, remove_images: bool = False) -> Dict[str, dict]: """Every check on every dataset (or ``dataset``); with ``write``, what passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the @@ -254,13 +383,26 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = the units check all of them, and each record says which shard. ``remove_images``: each unit's own image is removed after it, so the oracle pulls it again (by default they stay). Returns - {dataset: {check: record}}, each record with ``ok`` and its ``problems``.""" + {dataset: {check: record}}, each record with ``ok`` and its ``problems``. + ``jobs``: {control: job directory}, held to its bound instead of run.""" env = env if env is not None else os.environ stamp = stamp or time.strftime(STAMP) + jobs = dict(jobs or {}) sets = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not sets: raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})] + if jobs: + if dataset is None and len(sets) > 1: + raise run.Refused("a control's job is a run on one dataset: name it (--dataset)") + if limit or (shard and shard[1] > 1): + raise run.Refused("a control's job is held over the whole dataset: no limit, no shard") + for c, d in jobs.items(): + if c not in declared: + raise run.Refused(f"{bench['label']} declares no {c} control") + if not os.path.isdir(d): + raise run.Refused(f"{d}: no such job") + opts = {ds["id"]: options_for(bench, ds, bench_options) for ds in sets} if not declared: say(f"{bench['label']} declares no controls: only its units and nothing else can be checked") got: Dict[str, dict] = {} @@ -270,11 +412,11 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = checks: Dict[str, dict] = {} try: try: - driver = _Driver(bench, ds, work, licences=licences) + driver = _Driver(bench, ds, work, licences=licences, options=opts[ds["id"]]) except install.InstallFailed as e: raise run.Refused(f"could not install what the proof needs: {e}") checks["units"] = units_check(driver, ds) - ids = checks["units"].pop("ids") + every = ids = checks["units"].pop("ids") if shard and shard[1] > 1: ids = [u for _, u in shards.select(ids, *shard)] ids = ids[:limit] if limit else ids @@ -285,10 +427,18 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = finally: shutil.rmtree(work, ignore_errors=True) for c in declared: - checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, - model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, - shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, - remove_images=remove_images) + if not checks["units"]["ok"] and not jobs.get(c): + checks[c] = {"ok": False, "units": 0, "source": definitions.control(bench, c)["source"], + "problems": [UNITS_FAILED + ": the control was not run"]} + else: + checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, + model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, + shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, + options=opts[ds["id"]], job_dir=jobs.get(c), ids=every, + remove_images=remove_images) + if not checks["units"]["ok"]: + checks[c]["problems"].append(UNITS_FAILED + ": a job's cover is held to the driver's ids, which did not pass") + checks[c]["ok"] = False _say(say, c, checks[c]) total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"] for name, r in checks.items(): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index b96dfb7..b6a6a73 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -126,9 +126,10 @@ def _bench_page(b: dict) -> dict: # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], "reference_agent": x.get("reference_agent", ""), - "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), - **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), - "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} + "comparable": k == "original" and b["setups"]["original"].get("comparable", True) is not False, "units": (b["setups"]["original"].get("units") or b.get("units") or {}), + **({"models": definitions.paper_models(x), "default": x["default"], "providers": list(x["providers"]), + "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in definitions.paper_models(x)} + for p, pre in x["providers"].items()}} if "models" in x else {"any": True, "providers": list(x["providers"])})} for k, x in (b.get("setups") or {}).items() if k == "original"], # its own options (a search, a mode): each value with its line and what it needs; and the host requirements @@ -339,6 +340,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "Choose a model.") continue method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" + if method == "original": + picked = definitions.upstream_name(orig, picked, provider_) o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") @@ -415,8 +418,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()): add("model", "No paper models are listed for this benchmark; choose Any of your choice.") elif model_choice == "paper" and needs_model and orig.get("models"): - if recipe.get("model") not in orig["models"]: - add("model", "Choose one of the paper's listed models.") + if recipe.get("model") not in definitions.paper_models(orig): + add("model", "Choose one of the paper's listed models, or Any of your choice for another of the original's.") elif provider_ not in orig.get("providers", {}): add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 78c9c06..76fa01d 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -13,10 +13,12 @@ apart from the rest (:func:`reply_facts`), the reply also says, when it does, the serving provider that answered (``served_by``), how it ended (``end``) and an error it holds though its status was a success (``error``); reading -them never changes what the caller receives, nor the rest of the record. The +them never changes what the caller receives, nor the rest of the record but a failed Responses object's cost. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the -user's price list, ``nullius.prices``), marked as an estimate. +user's price list, ``nullius.prices``), marked as an estimate. A Responses object answered 200 that failed (its ``end``) with no usage +costs 0, marked as failed: the provider keeps no generation for it, and a +key's usage compared with the relay's sum would show one billed after all. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -41,6 +43,8 @@ """ from __future__ import annotations +import base64 +import binascii import hashlib import http.client import json @@ -77,6 +81,16 @@ # its next use, before any of the reply is read: worth one new connection. _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError) ANTHROPIC_VERSION = "2023-06-01" +ROUTE_REFUSED = (404, 503) +ROUTE_ERROR = "ApiProviderRouteError" +CONTEXT_ERROR = "ContextWindowExceededError" +STOP = "stop" +OVERFLOW = "context window" +COMPLETION = re.compile(r"^/(?:chat/completions|completions|messages|responses)$") +_WHOLE = ((b"\x89PNG\r\n\x1a\n", b"IEND\xaeB`\x82", "image/png"), (b"\xff\xd8\xff", b"\xff\xd9", "image/jpeg"), + (b"GIF87a", b";", "image/gif"), (b"GIF89a", b";", "image/gif")) +_PROVIDER = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._()/-]{0,79}") +_CONTEXT = re.compile(r"context[ _-]?(length|window|limit)|prompt is too long|token count .{0,40}exceeds", re.I) def _cut(text, n=4000): @@ -114,7 +128,7 @@ def _usage(u: Optional[dict]) -> Optional[dict]: inp += cached + written else: written = (det or {}).get("cache_write_tokens") or 0 - got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0} + got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or (rdet or {}).get("thinking_tokens") or 0} if written: got["cache_write"] = written hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0 @@ -148,6 +162,25 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def _hollow(path: str, body: bytes) -> Optional[tuple]: + """A 200 to a model call that holds no reply: the status it stands for, 429 or 402 when an error it held says so, else + 502, and its kind alone, since its words can quote the request.""" + if not COMPLETION.search(path.split("?")[0]): + return None + try: + got = json.loads(body) + except ValueError: + return 502, "no reply" + if not isinstance(got, dict): + return 502, "no reply" + if read_reply(path, body, False): + return None + code = got["error"].get("code") if isinstance(got.get("error"), dict) else None + code = code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code < 600 else None + return ((code if code in (402, 429) else 502), "an error" + (f" {code}" if code else "")) if got.get("error") \ + else (502, "no reply") + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: """What a reply said: its text, its tool calls, its usage.""" if not streamed: @@ -202,7 +235,7 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): t, c = _text_of_responses(ev.get("response") or {}) return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: @@ -417,6 +450,23 @@ def reply_facts(path: str, body: bytes, streamed: bool) -> dict: return {} +def _plain(name) -> str: + """A provider's name or slug, compared: lower case, letters and digits.""" + return re.sub(r"[^a-z0-9]", "", str(name).split("/")[0].lower()) + + +def held(provider: str, only: List[str]) -> Optional[bool]: + """True when the provider that answered is one of ``only``, False when it shares nothing with them, None when + it cannot be told: a name is not always its slug.""" + name = _plain(provider) + slugs = {_plain(p) for p in only} + if not name or name in slugs: + return True if name else None + if any(s.startswith(name) or name.startswith(s) for s in slugs if s): + return None + return False + + def _request_tail(req: dict) -> dict: """The part of a request a reader needs: the model, how many messages, the last two, abridged, and the reasoning effort it asked for (chat's @@ -442,6 +492,37 @@ def _request_tail(req: dict) -> dict: "reasoning_effort": effort if isinstance(effort, str) else None} +def _image(data: str) -> Optional[str]: + """An inline image's type when its bytes show a whole one: its head, and its end after it.""" + try: + raw = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + return None + if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP": + return "image/webp" if int.from_bytes(raw[4:8], "little") + 8 <= len(raw) else None + if raw[:2] == b"BM": + return "image/bmp" if int.from_bytes(raw[2:6], "little") <= len(raw) else None + return next((t for head, tail, t in _WHOLE if raw.startswith(head) and tail in raw[len(head):]), None) + + +def _images(x) -> List[dict]: + """Each inline image a request carries, as Messages or chat sends one: its declared type, the type its bytes + show whole (None when they show none) and its size, never a byte of it.""" + if isinstance(x, list): + return [i for v in x for i in _images(v)] + if not isinstance(x, dict): + return [] + src, url = x.get("source"), x.get("image_url") + url = url.get("url") if isinstance(url, dict) else url + if x.get("type") == "image" and isinstance(src, dict) and src.get("type") == "base64" and isinstance(src.get("data"), str): + said, data = src.get("media_type"), src["data"] + elif isinstance(url, str) and url.startswith("data:") and ";base64," in url: + said, data = url[5:].split(";", 1)[0], url.split(";base64,", 1)[1] + else: + return _images(list(x.values())) + return [{"type": str(said)[:40] if said else None, "is": _image(data), "bytes": len(data)}] + + def _digest(obj) -> str: text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True) return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16] @@ -498,8 +579,9 @@ class _Pool: KEEP = 16 # idle connections kept per provider; more are closed - def __init__(self, base: str): + def __init__(self, base: str, timeout: float = TIMEOUT): u = urllib.parse.urlsplit(base) + self.timeout = timeout self.https = u.scheme == "https" self.host, self.port = u.hostname, u.port or (443 if self.https else 80) self.prefix = u.path.rstrip("/") @@ -515,7 +597,7 @@ def take(self, fresh: bool = False): kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection with self.lock: self.opened += 1 - return kind(self.host, self.port, timeout=TIMEOUT), False + return kind(self.host, self.port, timeout=self.timeout), False def give(self, conn, r): """Back to the pool when the reply was read whole and the provider @@ -562,7 +644,7 @@ class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, - facts: Optional[bool] = None): + facts: Optional[bool] = None, timeout: float = TIMEOUT): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -585,7 +667,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s put on every such request in place of the caller's own. ``facts``: a pinned Chat Completions or Responses request asks OpenRouter for its routing facts, where its reply names who served it; by default when - ``base`` is OpenRouter's.""" + ``base`` is OpenRouter's. ``timeout``: the seconds one call may wait.""" self.fields = fields self.serving = {m: list(s) for m, s in (serving or {}).items()} host = (urllib.parse.urlsplit(base).hostname or "").lower() @@ -601,7 +683,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s self.models: Dict[str, str] = {} self.embedding_models: Dict[str, str] = {} self.lock = threading.Lock() - self.pool = _Pool(self.base) + self.pool = _Pool(self.base, timeout) self.server: Optional[ThreadingHTTPServer] = None os.makedirs(record_dir, exist_ok=True) @@ -671,8 +753,13 @@ def _headers(self, caller, facts: bool = False) -> Dict[str, str]: out["Authorization"] = "Bearer " + self.key return out + def _asked(self) -> bool: + return asked(self.dir) is not None + def _forward(self, h: BaseHTTPRequestHandler): - """One call of a caller, ``h``: sent on, passed back, recorded.""" + """One call of a caller, ``h``: sent on, passed back, recorded. A caller's own call + refused for its context window also leaves STOP beside the records; one made once + another asked by STOP is refused unsent and unrecorded.""" bits = h.path.split("/", 2) if len(bits) < 3 or not _TAG.match(bits[1]): h.send_error(404, "no route: //v1/...") @@ -704,6 +791,9 @@ def _forward(self, h: BaseHTTPRequestHandler): routing_asked = req.get("provider") req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if not side and not self.models.get(tag) and self._asked(): + h.send_error(400, "the unit was asked to stop") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -724,6 +814,7 @@ def _forward(self, h: BaseHTTPRequestHandler): t0 = time.time() conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed + cut_off = None try: # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) @@ -744,10 +835,17 @@ def _forward(self, h: BaseHTTPRequestHandler): except Exception as e: # cut off mid-reply status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() + empty = _hollow(rest, body) if status == 200 and not streamed and h.command == "POST" \ + and not (isinstance(req, dict) and req.get("stream")) else None + if empty: + (status, inside), kind = empty, "application/json" + back = [(n, v) for n, v in back if n.lower() in ("retry-after", "retry-after-ms")] + body = json.dumps({"error": {"message": f"the endpoint answered 200 with {inside}"}}).encode() def record(body): rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None, - "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {}, + "model_asked": asked, "status": status, + "request": dict(_request_tail(req), sha256=_digest(req)) if isinstance(req, dict) else {}, "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True @@ -759,7 +857,7 @@ def record(body): if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None - if status == 200: + if status == 200 or cut_off: rec["reply"] = read_reply(rest, body, streamed) # who served it, how it ended, an error under the 200: beside what was read, never in its place try: @@ -767,16 +865,25 @@ def record(body): if k not in rec["reply"]}) except Exception: pass - cost, how = self._cost(rec["model"], rec["reply"].get("usage")) + cost, how = (0.0, "failed") if rec["reply"].get("end") == "failed" and not rec["reply"].get("usage") \ + else self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - else: - rec["error"] = _said(status, body) + if status != 200: + rec["error"] = cut_off or _said(status, body) + sent = _images(req) if status in (400, 413, 422) else [] + if sent: + rec["images"] = sent + if empty: + rec["answered"] = 200 if booking is not None: rec["booked_usd"] = booked try: - if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) + if status == 200 or cut or empty: # what it cost; unknown (no token count, cut off), the booking is kept whole + whole = not (cut_off or rec.get("cost_from") == "failed") + known = cost if whole or (cost is not None and cost > booked) else None + self.ledger.settle(booking, known, rec["model"], + ((rec.get("reply") or {}).get("usage") or {}).get("output") if whole else None) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -784,6 +891,9 @@ def record(body): with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") + if not side and not self.models.get(tag) and overflowed(rec): + with open(os.path.join(self.dir, STOP), "a") as f: + f.write(OVERFLOW + "\n") h.send_response(status) if not any(n.lower() == "content-type" for n, _ in back): @@ -799,6 +909,8 @@ def record(body): while True: chunk = r.read1(65536) if not chunk: + if r.length: + raise http.client.IncompleteRead(b"", r.length) whole = True break parts.append(chunk) @@ -807,8 +919,8 @@ def record(body): h.wfile.flush() except OSError: pass # the caller went away; the record still completes - except Exception: - pass + except Exception as e: + status, cut, cut_off = 502, True, "the endpoint's reply was cut off: " + self._said(e) record(b"".join(parts)) # before the connection closes: the caller's end of stream else: record(body) # before the caller has the reply, so a reader after it finds the call @@ -932,11 +1044,26 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True -def failure(calls: List[dict]) -> Optional[tuple]: +def asked(record_dir: str) -> Optional[str]: + """What a guard's STOP in ``record_dir`` says, or None: none there, or only the context window's.""" + try: + with open(os.path.join(record_dir, STOP)) as f: + said = next((ln.strip() for ln in f.read().splitlines() or [""] if ln.strip() != OVERFLOW), None) + except OSError: + return None + return None if said is None else said or STOP + + +def failure(calls: List[dict], words: bool = True, agent: bool = False, serving: Optional[Dict[str, List[str]]] = None) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's). A call + succeeded (a failure the agent recovered from is not the unit's). + ``words`` False: the status alone, never the provider's text, which can quote + a gated task. ``agent``: the agent's last call over the model's context + window is ``ContextWindowExceededError``, its own, whatever side call followed, and a side call's 200 does not + recover the agent's own. A pinned call (``serving``) no endpoint took is + ``ApiProviderRouteError``. Calls are taken in the order they began. A call the run's spending cap refused is the unit's wherever it falls: once one is refused, every later one is.""" cut = next((c for c in calls if c.get("refused") == "cap"), None) @@ -945,21 +1072,76 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or calls[-1].get("status") == 200: + calls = _begun(calls) + ok = [i for i, c in enumerate(calls) if c.get("status") == 200] + after, side = max((i for i in ok if not (agent and calls[i].get("side"))), default=-1), max(ok, default=-1) + calls = [c for i, c in enumerate(calls) if i > after and c.get("status") != 200 and not (c.get("side") and i < side)] + if not calls: return None + own = next((c for c in reversed(calls) if not c.get("side")), None) if agent else None + if own is not None and overflowed(own): + said = (own.get("error") or "") if words else ("the endpoint answered 400 to the agent's call, longer than the model's " + "context window; its words are not kept, the benchmark's tasks being gated") + return CONTEXT_ERROR, said last = calls[-1] - s, said = last.get("status"), last.get("error") or "" + s = last.get("status") + said = (last.get("error") or "") if words else f"the endpoint answered {s}; its words are not kept, the benchmark's tasks being gated" + if (serving or {}).get(last.get("model")) and s in ROUTE_REFUSED: + return ROUTE_ERROR, _refusal(last, said, serving) if s == 429: return "RateLimitError", said if s in (401, 403): return "AgentAuthenticationError", said - if s == 404 and "model" in said.lower(): + if s == 404 and "model" in (last.get("error") or "").lower(): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said return "ApiError", said +def _begun(calls: List[dict]) -> List[dict]: + timed = all(isinstance(c.get("t0"), (int, float)) for c in calls) + return sorted(calls, key=lambda c: c["t0"]) if timed else list(calls) + + +def overflowed(call: dict) -> bool: + """A 400 for a request longer than the model's context window, by the provider's words.""" + return call.get("status") == 400 and bool(_CONTEXT.search(call.get("error") or "")) + + +def _refusal(call: dict, said: str, serving: Dict[str, List[str]]) -> str: + """A pinned route's refusal: its status first (route_dead reads it), the model, its pin, then ``said``.""" + text = f"{call.get('status')}: no endpoint of {call.get('model')} pinned to {', '.join(serving[call.get('model')])} took the call" + return f"{text}: {said}" if said else text + + +def routes_failure(routes: List[List[dict]], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """:func:`route_failure` of each route's calls on its own, the first found.""" + return next(filter(None, (route_failure(calls, serving, words) for calls in routes)), None) + + +def route_dead(exc: Optional[tuple]) -> bool: + """A pinned route with no endpoint at all (404): every later unit would fail the same way.""" + return bool(exc) and exc[0] == ROUTE_ERROR and str(exc[1]).startswith("404: ") + + +def route_failure(calls: List[dict], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """A pinned route (``serving``) that failed a unit, among one route's ``calls``: a model whose last call was + refused, or a call its reply says another provider served, by a name that reads as one.""" + last = {} + for c in _begun(calls): + last[c.get("model")] = c + by = (c.get("reply") or {}).get("served_by") + if serving.get(c.get("model")) and isinstance(by, str) and _PROVIDER.fullmatch(by) and held(by, serving[c.get("model")]) is False: + return ROUTE_ERROR, f"{c.get('status')}: {c.get('model')} pinned to {', '.join(serving[c.get('model')])} was served by {by}" + for c in last.values(): + if serving.get(c.get("model")) and c.get("status") in ROUTE_REFUSED: + said = (c.get("error") or "") if words else \ + f"the endpoint answered {c.get('status')}; its words are not kept, the benchmark's tasks being gated" + return ROUTE_ERROR, _refusal(c, said, serving) + return None + + # -- the relay as a process of its own: a container backend's sidecar ---------- def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL @@ -992,6 +1174,7 @@ def serve(argv=None) -> int: "or --routing given") ap.add_argument("--routing", action="store_true", help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") + ap.add_argument("--timeout", type=float, default=TIMEOUT, help="seconds one call may wait for the provider's reply") a = ap.parse_args(argv) serving, bad = parse_served_by(a.served_by, "") if serving and not bad: @@ -1016,7 +1199,8 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, + timeout=a.timeout) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 4621e54..bd9d9c6 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -51,6 +51,7 @@ """ from __future__ import annotations +import fcntl import hashlib import json import os @@ -66,7 +67,7 @@ from collections import Counter from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from contextlib import contextmanager -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact @@ -114,6 +115,10 @@ def driver_path() -> str: _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) +HELD: dict = {} +LIVE: dict = {} + + class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" @@ -360,7 +365,37 @@ def get(self, name: str) -> Optional[dict]: return {"kind": kind, "value": text} if text else None +def resolve_bench_options(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + which=shutil.which) -> Tuple[dict, dict]: + """The benchmark's options for a run on ``dataset`` and what their values need; refused before anything starts.""" + vals, bad = definitions.bench_options_of(bench, dataset, given, setup) + if bad: + raise Refused("; ".join(bad)) + needs = definitions.option_needs(bench, vals) + lacking = hostreq.problems(bench, needs["host"], which) + if lacking: + raise Refused(f"{', '.join(f'{k}={v}' for k, v in vals.items())} needs on this machine " + + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) + else ". Run it on a machine that has it")) + return vals, needs + + +def pins_of(bench: dict, other: dict, provider: str, judge_model: Optional[str]) -> Dict[str, List[str]]: + """{model as sent: [provider slug]}: the original setup's pins and the official judge's, on a provider that pins.""" + if provider not in definitions.PIN_PROVIDERS: + return {} + pins = {m: list(o) for m, o in (other.get("pins") or {}).items()} if other.get("_kind") == "upstream" else {} + j = (bench.get("score") or {}).get("judge") or {} + only = (j.get("pin") or {}).get(provider) + if only and judge_model and judge_model == providers.served_as(j["model"], provider): + pins[judge_model] = list(only) + return pins + + class Run: + call_seconds: float = relay.TIMEOUT + def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -562,6 +597,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas if stray: raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + (", ".join(called) or "no model")) + self.route_pins = pins_of(bench, other, provider, self.judge_model) + self.serving.update(self.route_pins) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -577,6 +614,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + self.score_seconds = (bench.get("limits") or {}).get("score_seconds") or DRIVER_SECONDS + self.call_seconds = (bench.get("limits") or {}).get("call_seconds") or relay.TIMEOUT self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -644,8 +683,25 @@ def driver(self) -> jsonl.Proc: return d def start(self) -> str: + lck, mine = os.path.join(self.job_dir, ".work", "job.lck"), object() + try: + return self._start(lck, mine) + finally: + if LIVE.get(lck) is mine: + del LIVE[lck] + + def _start(self, lck, mine) -> str: t0 = time.time() os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True) + held = HELD.get(lck) or open(lck, "a") + try: + if LIVE.setdefault(lck, mine) is not mine: + raise BlockingIOError + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused(f"another run under the job name {self.name} is running: " + "start this one once it ends, or under another name") from None + HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) @@ -844,11 +900,12 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, job's ``retried/`` (kept whole, its calls and its cost with it, where no reader takes it for a trial of its own), its second written under the trial's own name and counted (into ``done``, as each ends). Nothing - is run again once the run is stopped or halted; the spending cap goes - on counting every call.""" + is run again once the run is stopped or halted, nor a unit a guard's + stop asked to stop; the spending cap goes on counting every call.""" by_name = {self.trial_name(*iur): iur for iur in work} todo = [by_name[r["trial_name"]] for r in results - if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name] + if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name + and relay.asked(os.path.join(self.job_dir, r["trial_name"], ".work", "relay")) is None] if not todo: return self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time") @@ -954,7 +1011,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False, - serving=self.serving, + serving=self.serving, timeout=self.call_seconds, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -993,6 +1050,11 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home") + earlier = os.path.join(trial, ".work", f"earlier.{time.time_ns()}") + for part in (os.path.join(".work", "relay"), "result.json", "verifier", "agent"): + if os.path.lexists(os.path.join(trial, part)): + os.makedirs(earlier, exist_ok=True) + os.replace(os.path.join(trial, part), os.path.join(earlier, os.path.basename(part))) os.makedirs(ws, exist_ok=True) # The workspace is a repository of its own: agents look up to the git root # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept @@ -1065,17 +1127,17 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.score_seconds + (float(step.get("seconds") or self.turn_seconds) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.score_seconds, **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e)) + raise _Ended(_kind(e, "VerifierError"), str(e)) except _Ended as e: exc = (e.kind, e.said) traj = traj or e.trajectory @@ -1094,17 +1156,23 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: for ln in getattr(rl, "mcp_listeners", None) or []: ln.stop() calls = rl.calls(tag) if rl is not None else [] - failed = relay.failure(calls) + words = not self.bench.get("gated") + failed = relay.failure(calls, words, agent=True, serving=getattr(self, "route_pins", None)) if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is # the endpoint's, and a cap's refusal is the cap's failed = None - if not failed and rl is not None and exc is not None and exc[0] == "VerifierError": + if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") + or self.upstream and exc and ": VerifierError: " in exc[1]): + failed = None + if not failed and rl is not None and getattr(self, "route_pins", None): + failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) + if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): # the judge's or the simulated user's model failing (a 429, a refused request) is the # unit's infrastructure, as the agent's own would be - not a miss counted against it - failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) + failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}"), words) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score = None @@ -1205,7 +1273,7 @@ def note_refusal(self, exc: Optional[tuple]) -> None: if getattr(self, "_halt", None) is None: return with self._said: - if exc and exc[0] in REFUSALS: + if exc and (exc[0] in REFUSALS or relay.route_dead(exc)): self._refused.append(exc) if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None: self.halted = exc @@ -1353,7 +1421,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1383,6 +1452,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds if self.call_seconds != relay.TIMEOUT else None, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1901,8 +1971,8 @@ def send(op, **kw): def drive_upstream(self, unit, ws, tag, transcript, rl): """The setup's unit: the driver runs the upstream's own answerer on it (``answer``), asked as ``asked`` through the unit's relay, which sends - the run's model in its place. A reply with no text - a reasoning model - that spent the upstream's token budget thinking - is said as such.""" + the run's model in its place. A last reply with no text or tool call - a reasoning model + that spent the upstream's token budget thinking - is said as such; a unit a guard stopped is cancelled.""" o = self.other try: # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes @@ -1911,7 +1981,18 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) except jsonl.Failed as e: calls = rl.calls(tag) - silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] + stop = relay.asked(rl.dir) + if stop and not _kind(e, None): + raise _Ended(CANCELLED, f"a guard asked the unit to stop ({stop[:200]}); {o['label']}: {e}", + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + own = [c for c in relay._begun(calls) if not c.get("side")][-1:] + failed = [((c["reply"].get("error") or {}).get("code") or "no code") for c in own if (c.get("reply") or {}).get("end") == "failed"] + kind = _kind(e, "ApiError" if failed else None) + if kind: + raise _Ended(kind, f"{o['label']}: {e}" + "".join(f"; {self.model}'s last reply failed ({c})" for c in failed), + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + silent = [c for c in own if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip() + and not (c.get("reply") or {}).get("tool_calls")] if silent: u = (silent[-1].get("reply") or {}).get("usage") or {} thought = u.get("reasoning") or 0 @@ -1926,6 +2007,12 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") +def _kind(e, otherwise): + """A driver's own EnvironmentSetupError, VerifierError, CancelledError or CauseUnknownError keeps its kind.""" + m = re.match(r"\w+: (EnvironmentSetupError|VerifierError|CancelledError|CauseUnknownError): ", str(e)) + return m.group(1) if m else otherwise + + class _Ended(Exception): def __init__(self, kind, said, trajectory=None): super().__init__(said) diff --git a/app/tests/test_dependency_surface.py b/app/tests/test_dependency_surface.py index 12b6b2e..d6cbd84 100644 --- a/app/tests/test_dependency_surface.py +++ b/app/tests/test_dependency_surface.py @@ -9,6 +9,7 @@ viewer independent of the runtime. """ +import fnmatch import importlib.util import json import os @@ -80,6 +81,26 @@ def test_every_backend_package_is_shipped_once(self): self.assertEqual({p for p in packages if p == "nullius" or p.startswith("nullius.")}, source) self.assertEqual({p.split(".")[0] for p in packages}, {"nullius", "_nullius_app"}) + def test_every_file_a_bundled_definition_runs_is_shipped(self): + """Every file a host's snapshot takes of a bundled entry is package data too.""" + from nullius.standard import payload + shipped = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["setuptools"]["package-data"] + + def covered(package, relative): + parts = relative.split("/") + return any(len(g.split("/")) == len(parts) and all(map(fnmatch.fnmatchcase, parts, g.split("/"))) + for g in shipped[package]) + + base = APP / "nullius" + entries = [("nullius.catalogue", f"{kind}/{e.name}", payload._entry(e)) + for kind in ("agents", "benchmarks", "capabilities") + for e in sorted((base / "catalogue" / kind).iterdir()) if e.is_dir() and e.name != "__pycache__"] + entries += [("nullius.standard", f"kits/{e.name}", payload._entry(e, kit=True)) + for e in sorted((base / "standard" / "kits").iterdir()) if (e / "standard.json").is_file()] + missing = [f"{where}/{name}" for package, where, files in entries for name in files + if not covered(package, f"{where}/{name}")] + self.assertEqual(missing, []) + def test_every_requirement_is_on_the_allowlist(self): for req in requirements(): with self.subTest(req): diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py new file mode 100644 index 0000000..66c6d94 --- /dev/null +++ b/app/tests/test_standard_context_window.py @@ -0,0 +1,287 @@ +"""An agent's last call over the model's window ends the unit as ContextWindowExceededError, whatever followed.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from nullius.runs import classify, parse +from nullius.standard import definitions, install, job, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +WORDS = "the gated task's own sentence" +OPENROUTER = f"HTTP 400: This endpoint's maximum context length is 1000 tokens. However, you requested 1200: {WORDS}" +ANTHROPIC = f"HTTP 400: prompt is too long: 1200 tokens > 1000 maximum: {WORDS}" +EMPTY = "HTTP 400: messages: at least one message is required" +AGENT_OK = {"status": 200} +OVERFLOW = {"status": 400, "error": OPENROUTER} +SIDE_EMPTY = {"status": 400, "error": EMPTY, "side": True} +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's runner ended on its simulated user's error") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' +STOPPED = ANSWERING.replace('raise RuntimeError("the upstream\'s runner ended on its simulated user\'s error")', + 'return {"answer": "DONE"}').replace('{"reward": 0.0, "detail": {}}', + '{"reward": 0.5, "detail": {}, "stopped": "context window"}') +UNREAD = '''from nullius.standard.jsonl import serve +class VerifierError(RuntimeError): + pass +def score(unit, dir, **_): + raise VerifierError("the evaluation did not read the VM") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": lambda unit, dir, setup, model, base_url, key, **_: {"answer": "DONE"}, "score": score}) +''' +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Failure(unittest.TestCase): + def test_the_agents_overflow_outranks_the_side_call_after_it(self): + calls = [AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, agent=True), (relay.CONTEXT_ERROR, OPENROUTER)) + exc, said = relay.failure(calls, words=False, agent=True) + self.assertEqual(exc, "ContextWindowExceededError") + self.assertIn("answered 400", said) + self.assertIn("context window", said) + self.assertNotIn(WORDS, said) + self.assertNotIn("maximum context length", said) + + def test_anthropics_words_and_a_later_rate_limit(self): + calls = [AGENT_OK, {"status": 400, "error": ANTHROPIC}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_the_agents_overflow_as_the_last_call(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW], words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_an_overflow_the_agent_recovered_from_is_not_the_units(self): + self.assertIsNone(relay.failure([OVERFLOW, AGENT_OK], agent=True)) + + def test_a_side_call_that_went_through_after_it_does_not_recover_the_agent(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW, {"status": 200, "side": True}], agent=True)[0], relay.CONTEXT_ERROR) + limited = [AGENT_OK, {"status": 429, "error": "slow down"}, {"status": 502, "error": "x"}, + {"status": 429, "error": "slow down"}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(limited, words=False, agent=True)[0], "RateLimitError") + self.assertIsNone(relay.failure(limited[1:3] + [AGENT_OK, {"status": 200, "side": True}], agent=True)) + self.assertIsNone(relay.failure(limited, words=False)) + calls = [AGENT_OK, OVERFLOW, {"status": 200, "side": True}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + self.assertEqual(relay.failure(calls, words=False)[0], "RateLimitError") + calls = [{"status": 429, "error": "x"}, {"status": 502, "error": "x", "side": True}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "RateLimitError") + + def test_the_calls_are_read_in_the_order_they_began(self): + abandoned = {"status": 502, "error": "x", "t0": 1} + self.assertIsNone(relay.failure([dict(AGENT_OK, t0=2), abandoned], agent=True)) + self.assertEqual(relay.failure([{"status": 429, "error": "x", "t0": 2}, dict(AGENT_OK, t0=1)])[0], "RateLimitError") + self.assertEqual(relay.failure([dict(AGENT_OK, t0=1), {"status": 429, "error": "x"}], agent=True)[0], "RateLimitError") + + def test_another_400_of_the_agent_stays_the_infrastructures(self): + calls = [AGENT_OK, {"status": 400, "error": "HTTP 400: invalid image"}, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_side_calls_overflow_is_not_the_agents(self): + calls = [AGENT_OK, dict(OVERFLOW, side=True)] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_judges_or_simulators_route_is_never_the_agents_context(self): + calls = [AGENT_OK, OVERFLOW] + self.assertEqual(relay.failure(calls, words=False)[0], "ApiError") + self.assertEqual(relay.failure(calls)[0], "ApiError") + + def test_only_a_400_says_so(self): + self.assertTrue(relay.overflowed(OVERFLOW)) + self.assertTrue(relay.overflowed({"status": 400, "error": ANTHROPIC})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: input length and `max_tokens` exceed context " + "limit: 196000 + 8192 > 200000"})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: The input token count (1200) exceeds the " + "maximum number of tokens allowed (1000)."})) + self.assertFalse(relay.overflowed(dict(OVERFLOW, status=500))) + self.assertFalse(relay.overflowed({"status": 400, "error": EMPTY})) + self.assertFalse(relay.overflowed({"status": 400})) + + def test_it_is_the_agents_result_not_the_infrastructure(self): + self.assertEqual(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.AGENT_ERROR) + self.assertNotIn(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.INFRA_OUTCOMES) + fed = job.feed_outcome("score", None, (relay.CONTEXT_ERROR, "said")) + self.assertEqual((fed["outcome"], fed["exc"]), ("agent_error", relay.CONTEXT_ERROR)) + + +class Calls: + """The agent's calls ending on an overflow, then a refused side call.""" + def __init__(self, own): + self.own, self.dir = own, os.devnull + + def calls(self, tag): + return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Unit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "paper") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, setups=SETUPS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def ran(self, own, gated=True): + b = dict(self.b, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f).get("exception_info"), parse.parse_job(job_dir) + + def test_the_unit_is_the_agents_context_said_by_status_alone(self): + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn("answered 400", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + (trial,) = ran.trials + self.assertEqual(trial.outcome, classify.AGENT_ERROR) + self.assertFalse(trial.is_infra) + self.assertEqual(ran.n_infra, 0) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY], gated=False) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn(WORDS, said["exception_message"]) + + def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW]) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD) + said, ran = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "VerifierError") + self.assertIn("the evaluation did not read the VM", said["exception_message"]) + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD.replace("raise VerifierError", "raise RuntimeError")) + said, _ = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + + def test_a_unit_run_again_under_its_jobs_name_starts_with_no_earlier_record_and_keeps_a_new_stop(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + record = os.path.join(jobs, "again", "a__u0000", ".work", "relay") + os.makedirs(record) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "verifier")) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "agent")) + for name, text in (("u0000.jsonl", json.dumps({"status": 429, "error": "slow down"}) + "\n"), (relay.STOP, "batch cap\n"), + ("../../verifier/score.json", "{}"), ("../../result.json", "{}"), ("../../agent/old.txt", "")): + with open(os.path.join(record, name), "w") as f: + f.write(text) + seen = [] + + def phase(r, name, unit, phase, timeout=None): + if phase == "env_setup": + os.makedirs(record, exist_ok=True) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("key cap\n") + + def unit_relay(r, tag, trial, *a, **k): + with open(os.path.join(trial, ".work", "relay", relay.STOP)) as f: + seen.append((sorted(os.listdir(os.path.join(trial, ".work", "relay"))), f.read())) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "phase", phase), patch.object(run.Run, "unit_relay", unit_relay): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + self.assertEqual(seen, [([relay.STOP], "key cap\n")]) + (earlier,) = [d for d in os.listdir(os.path.dirname(record)) if d.startswith("earlier.")] + earlier = os.path.join(os.path.dirname(record), earlier) + with open(os.path.join(earlier, "relay", relay.STOP)) as f: + self.assertEqual(f.read(), "batch cap\n") + for part in (("verifier", "score.json"), ("result.json",), ("agent", "old.txt")): + self.assertTrue(os.path.isfile(os.path.join(earlier, *part))) + self.assertFalse(os.path.exists(os.path.join(jobs, "again", "a__u0000", "verifier", "score.json"))) + + def test_a_run_under_a_job_name_another_run_holds_is_refused_before_it_writes(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + job_dir = os.path.join(jobs, "again") + lck = os.path.join(job_dir, ".work", "job.lck") + tried = subprocess.run([sys.executable, "-c", "import fcntl, sys\nfcntl.flock(open(sys.argv[1], 'a'), fcntl.LOCK_EX | fcntl.LOCK_NB)", lck], + capture_output=True) + self.assertNotEqual(tried.returncode, 0) + run.HELD.pop(lck).close() + before = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time\nf = open(sys.argv[1], 'a')\n" + "fcntl.flock(f, fcntl.LOCK_EX)\nprint(flush=True)\ntime.sleep(30)", + lck], stdout=subprocess.PIPE) + self.addCleanup(holder.stdout.close) + self.addCleanup(holder.kill) + holder.stdout.readline() + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])), \ + self.assertRaisesRegex(run.Refused, "another run under the job name again is running"): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + after = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + self.assertEqual(after, before) + + def test_a_second_run_in_the_same_process_under_a_live_job_name_is_refused(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + go = lambda: run.run(self.b, o, model="paper-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + name="live", say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + tried = [] + + def relay(r, *a, **k): + if not tried: + tried.append(None) + for again in (go, r.start): + try: + tried.append(again()) + except run.Refused as e: + tried.append(e) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "unit_relay", relay): + go() + self.assertEqual(len(tried), 3) + for refused in tried[1:]: + self.assertIsInstance(refused, run.Refused) + self.assertIn("another run under the job name live is running", str(refused)) + go() + + def test_a_side_call_failing_after_the_agents_success_stays_the_infrastructures(self): + said, ran = self.ran([AGENT_OK, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ApiError") + self.assertEqual(ran.trials[0].outcome, classify.API_ERROR) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index e1c6b84..7bd8982 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -166,6 +166,101 @@ def test_a_standard_job_named_oracle_without_the_record_is_no_control(self): self.assertFalse(parse.parse_job(j.dir).is_control) # a runner's arm called oracle stays an arm +SLOW_SCORE = '''import os, time +from nullius.standard.jsonl import serve +CHECKED = os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "checked")) +def setup(unit, dir): + return dict({"prompt": "hi"}, **({"verify": {"run": "true"}} if CHECKED else {})) +def score(unit, dir, transcript, answer, verified=None): + time.sleep(2) + return {"reward": 0.0, "passed": False, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "score": score}) +''' + + +class Box: + image = "" + + def give_back(self, paths): + pass + + def stop(self): + pass + + +class ScoreLimit(unittest.TestCase): + """A null control whose driver does its work in ``score``, waited for ``limits.score_seconds``.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_SCORE) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def problems(self, limits): + return definitions.problems(dict(self.b, limits=limits), "benchmark") + + def trial(self, limits): + b = dict(self.b, limits=limits) + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", + jobs_root=os.path.join(self.tmp, "jobs"), env={}, say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f) + + def verify_phase(self): + jobs = os.path.join(self.tmp, "jobs") + with open(os.path.join(jobs, os.listdir(jobs)[0], "events.jsonl")) as f: + return next(e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "verify") + + def test_the_limit_is_a_whole_number_of_seconds_from_1(self): + self.assertEqual(self.problems({"score_seconds": 1800}), []) + for v in (0, -1, 1.5, "600", True, 100001): + self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): + r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) + self.assertEqual(r.score_seconds, 600) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}) + self.assertEqual(r.score_seconds, 1800) + + def test_a_score_past_the_limit_is_the_verifiers_error_and_one_within_it_scores(self): + late = self.trial({"score_seconds": 1}) + self.assertEqual(late["exception_info"]["exception_type"], "VerifierError") + self.assertIn("score: no reply in 1 s", late["exception_info"]["exception_message"]) + self.assertIsNone(late.get("verifier_result")) + on_time = self.trial({"score_seconds": 10}) + self.assertIsNone(on_time.get("exception_info")) + self.assertEqual(on_time["verifier_result"]["rewards"]["reward"], 0.0) + + def test_the_verify_phase_carries_the_score_limit_and_an_own_images_check(self): + self.trial({"score_seconds": 10}) + self.assertEqual(self.verify_phase()["timeout_sec"], 10) + shutil.rmtree(os.path.join(self.tmp, "jobs")) + open(os.path.join(self.b["_here"], "checked"), "w").close() + checked = [] + with patch.object(run.Run, "unit_environment", lambda r, setup: ("task", {"image": "own"})), \ + patch.object(run.Run, "not_run", lambda r, e, d: None), \ + patch.object(run.Run, "unit_relay", lambda r, *a, **k: None), \ + patch.object(run.Run, "own_box", lambda r, *a: Box()), \ + patch.object(run.Run, "box_changes", lambda r, box: ""), \ + patch.object(run.Run, "release_image", lambda r, image: None), \ + patch.object(run.Run, "verify", lambda r, step, box, trial: checked.append(step) or {"exit": 0}): + got = self.trial({"score_seconds": 10}) + self.assertIsNone(got.get("exception_info")) + self.assertEqual(checked, [{"run": "true"}]) + self.assertEqual(self.verify_phase()["timeout_sec"], 10 + run.TURN_SECONDS) + + class Prove(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() @@ -281,6 +376,254 @@ def test_units_that_may_not_be_redistributed_are_counted_never_named(self): self.assertIn("held-unit-2", b["proven"]["leaky"]["failed"]["leak"]["problems"][0]) +class ProveOptions(unittest.TestCase): + """The benchmark's options reach the checks' driver and each control's run; a merged control job is held as one.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "kits", "agent-task-v1") + shutil.copytree(os.path.join(definitions.KITS, "agent-task-v1"), here) + with open(os.path.join(here, "driver.py")) as f: + src = f.read() + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(src.replace('return {"units": list(WORDS)}', + 'import os\n return {"units": list(WORDS) if os.environ.get("NULLIUS_OPTION_SITES") == "on" else []}')) + with open(os.path.join(here, "standard.json")) as f: + d = json.load(f) + d["options"] = {"sites": {"summary": "The sites.", "values": ["off", "on"], "default": "off"}, + "lane": {"summary": "A lane.", "values": ["a", "b"], "default": "a"}, + "tool": {"summary": "A tool.", "values": ["none", "x"], "default": "none", + "needs": {"x": {"host": ["tool"]}}}} + d["host"] = {"tool": {"summary": "A program this machine lacks.", "check": ["nullius-no-such-program-k5"]}} + with open(os.path.join(here, "standard.json"), "w") as f: + json.dump(d, f) + self.b, bad = definitions._read(os.path.join(here, "standard.json"), "benchmark", "agent-task-v1") + assert not bad, bad + self.jobs = os.path.join(self.tmp, "jobs") + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def prove(self, **kw): + kw.setdefault("stamp", "2026-01-02__03-04-05") + return prove.prove(self.b, jobs_root=self.jobs, env={}, say=lambda s: None, **kw) + + def config(self, job_dir): + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"] + + def nop_shard(self, i, n, lane): + return run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=self.jobs, env={}, + say=lambda s: None, shard=(i, n), launch="L", bench_options={"sites": "on", "lane": lane}) + + def test_the_options_reach_the_units_driver_and_each_controls_run(self): + got = self.prove(bench_options={"sites": "on"}, write=False)["kit"] + self.assertTrue(all(r["ok"] for r in got.values()), got) + self.assertEqual((got["units"]["units"], got["nop"]["passed"], got["oracle"]["passed"]), (3, "0/3", "3/3")) + self.assertEqual(self.config(os.path.join(self.jobs, got["nop"]["job"]))["benchmark_options"], + {"sites": "on", "lane": "a", "tool": "none"}) + self.assertEqual(got["nop"]["options"], {"sites": "on", "lane": "a", "tool": "none"}) + dflt = self.prove(write=False, stamp="2026-01-02__03-04-06")["kit"] + self.assertEqual(dflt["units"]["problems"], ["units: no ids"]) + self.assertEqual(dflt["nop"]["problems"], ["the units check failed: the control was not run"]) + + def test_a_name_or_a_value_the_benchmark_does_not_allow_is_refused_before_anything_runs(self): + was = json.dumps(self.b.get("proven")) + for given, why in (({"sites": "maybe"}, "'maybe' is not one of off, on"), ({"nope": "x"}, "has no option 'nope'")): + with self.assertRaises(run.Refused) as e: + self.prove(bench_options=given) + self.assertIn(why, str(e.exception)) + self.assertFalse(os.path.exists(self.jobs)) + self.assertEqual(json.dumps(self.b.get("proven")), was) + + def test_an_option_whose_value_needs_a_program_this_machine_lacks_is_refused_before_any_install(self): + with patch.object(install, "upstream", side_effect=AssertionError("installed")) as up: + with self.assertRaises(run.Refused) as e: + self.prove(bench_options={"sites": "on", "tool": "x"}) + self.assertIn("nullius-no-such-program-k5 not on PATH", str(e.exception)) + up.assert_not_called() + self.assertFalse(os.path.exists(self.jobs)) + + def test_a_job_is_not_recorded_when_the_units_check_fails(self): + from nullius.standard import merge + whole = merge.merge([self.nop_shard(i, 4, "a") for i in (1, 3)], os.path.join(self.jobs, "nop-merged"))["out"] + self.b["datasets"][0]["units"] = 4 + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertFalse(got["units"]["ok"]) + self.assertFalse(got["nop"]["ok"]) + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", got["nop"]["problems"]) + kept = self.b["proven"]["kit"] + self.assertNotIn("nop", kept) + self.assertNotIn("units", kept) + self.assertEqual(kept["failed"]["nop"]["job"], "nop-merged") + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", + kept["failed"]["nop"]["problems"]) + + def test_a_control_is_not_run_nor_kept_when_the_units_check_fails(self): + self.assertIn("nop", self.b["proven"]["kit"]) + self.b["datasets"][0]["units"] = 4 + got = self.prove(bench_options={"sites": "on"})["kit"] + self.assertFalse(got["units"]["ok"]) + for c in ("nop", "oracle"): + self.assertEqual(got[c]["problems"], ["the units check failed: the control was not run"]) + self.assertNotIn(c, self.b["proven"]["kit"]) + self.assertIn(c, self.b["proven"]["kit"]["failed"]) + self.assertFalse(os.path.exists(self.jobs) and any("prove-" in x for x in os.listdir(self.jobs))) + + def test_a_trial_of_another_job_cannot_be_merged_into_a_proof(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, "a") for i in (1, 3)] + trial = next(t for t in sorted(os.listdir(shards[0])) if os.path.isfile(os.path.join(shards[0], t, "result.json"))) + path = os.path.join(shards[0], trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + with self.assertRaises(merge.Refused) as e: + merge.merge(shards, os.path.join(self.jobs, "never")) + self.assertIn("names another job than its job", str(e.exception)) + forced = merge.merge(shards, os.path.join(self.jobs, "forced"), force=True)["out"] + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": forced}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_a_merge_of_merges_keeps_what_any_of_them_was_merged_despite(self): + from nullius.standard import merge + good_shard, bad_shard = (self.nop_shard(i, 4, "a") for i in (1, 3)) + trial = next(t for t in sorted(os.listdir(bad_shard)) if os.path.isfile(os.path.join(bad_shard, t, "result.json"))) + path = os.path.join(bad_shard, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + good = merge.merge([good_shard], os.path.join(self.jobs, "good"))["out"] + bad = merge.merge([bad_shard], os.path.join(self.jobs, "bad"), force=True)["out"] + with self.assertRaises(merge.Refused): + merge.merge([good, bad], os.path.join(self.jobs, "never")) + both = merge.merge([good, bad], os.path.join(self.jobs, "both"), force=True)["out"] + std = self.config(both) + self.assertTrue(any(w.startswith("bad: ") and "not of the job they are in" in w for w in std["merged_despite"])) + self.assertEqual([m.get("merged_from", [{}])[0].get("job") for m in std["merged_from"]], [os.path.basename(good_shard), os.path.basename(bad_shard)]) + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": both}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_the_cli_hands_its_options_and_jobs_to_the_proof(self): + from nullius import progress + from nullius.standard import cli + seen = {} + with patch.object(cli, "_bench", return_value=self.b), patch.object(cli, "licences", return_value=(None, None)), \ + patch.object(cli, "_exit_on_term"), patch.object(progress, "ECHO", None), \ + patch.object(prove, "prove", side_effect=lambda b, **kw: seen.update(kw) or {}): + self.assertEqual(cli.main(["prove", "agent-task-v1", "--from-run", "--dataset", "kit", "--bench-option", "sites=on", + "--bench-option", "lane=b", "--job", "nop=/x/merged"]), 0) + self.assertEqual((seen["bench_options"], seen["jobs"]), ({"sites": "on", "lane": "b"}, {"nop": "/x/merged"})) + with self.assertRaises(SystemExit): + cli.main(["prove", "agent-task-v1", "--from-run", "--job", "reference=/x"]) + + def test_a_merged_launch_is_held_to_the_bound_as_one_job_and_must_cover_every_unit(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, lane) for i, lane in ((1, "a"), (3, "b"))] + whole = merge.merge(shards, os.path.join(self.jobs, "nop-merged"))["out"] + before = sorted(os.listdir(self.jobs)) + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertTrue(got["nop"]["ok"], got["nop"]) + self.assertEqual((got["nop"]["job"], got["nop"]["units"], got["nop"]["passed"]), ("nop-merged", 3, "0/3")) + self.assertEqual(self.b["proven"]["kit"]["nop"]["job"], "nop-merged") + self.assertEqual(self.b["proven"]["kit"]["nop"]["options"], + [{"lane": "a", "sites": "on", "tool": "none"}, {"lane": "b", "sites": "on", "tool": "none"}]) + self.assertEqual(sorted(set(os.listdir(self.jobs)) - set(before)), ["2026-01-02__03-04-05__agent-task-v1__kit__prove-oracle"]) + self.assertEqual([m["benchmark_options"]["lane"] for m in self.config(whole)["merged_from"]], ["a", "b"]) + half = merge.merge(shards[:1], os.path.join(self.jobs, "nop-half"))["out"] + part = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": half}, write=False, stamp="2026-01-02__03-04-06")["kit"]["nop"] + self.assertFalse(part["ok"]) + self.assertTrue(any("units have no trial" in p for p in part["problems"]), part["problems"]) + wrong = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"oracle": whole}, write=False, stamp="2026-01-02__03-04-07")["kit"]["oracle"] + self.assertIn("nop-merged: its control is 'nop', not 'oracle'", wrong["problems"]) + trial = next(t for t in sorted(os.listdir(whole)) if os.path.isfile(os.path.join(whole, t, "result.json"))) + with open(os.path.join(whole, trial, "result.json")) as f: + res = json.load(f) + res.pop("verifier_result", None) + res.pop("exception_info", None) + with open(os.path.join(whole, trial, "result.json"), "w") as f: + json.dump(res, f) + blank = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole}, write=False, + stamp="2026-01-02__03-04-08")["kit"]["nop"] + self.assertFalse(blank["ok"]) + self.assertIn("1 unit(s) ended with no reward from 0 to 1 and no error: read the job's trials", blank["problems"]) + for kw in ({"limit": 1}, {"shard": (1, 2)}): + with self.assertRaises(run.Refused): + self.prove(dataset="kit", jobs={"nop": whole}, **kw) + with self.assertRaises(run.Refused): + self.prove(jobs={"nop": os.path.join(self.tmp, "absent")}) + + +NUMERIC = {"key": "numeric", "_ref": "numeric", "label": "Numeric", "family": "agent", "needs": ["agent.task/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "score", "says": "a number from 0 to 1"}, + "controls": {"nop": {"source": "nullius", "from": "Nothing runs."}}, "datasets": [{"id": "all", "units": 3}]} + + +class HeldJob(unittest.TestCase): + """A control's job held to its bound, read from each trial's own record.""" + + def setUp(self): + from nullius.standard import job + self.tmp = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, self.tmp, True) + self.dir = os.path.join(self.tmp, "nop-job") + os.makedirs(self.dir) + nop = definitions.control(NUMERIC, "nop") + self.units = ["u1", "u2", "u3"] + job.write_config(self.dir, job="nop-job", bench=NUMERIC, other=nop, model="", provider="openai", dataset="all", + units=self.units, concurrency=1, granted=False) + results = [job.write_trial(os.path.join(self.dir, f"{u}__u{i:04d}"), job="nop-job", bench=NUMERIC, other=nop, model="", + unit=u, t0=1_760_000_000.0, t1=1_760_000_001.0, produce=(None, None), score={"reward": 0.0}, + exception=None, calls=[], trajectory=None) for i, u in enumerate(self.units)] + job.write_result(self.dir, "nop-job", 1_760_000_000.0, 1_760_000_002.0, results, 3) + + def held(self): + return prove.control_check(NUMERIC, NUMERIC["datasets"][0], "nop", limit=None, jobs_root=self.tmp, stamp="s", + provider="openai", model="", judge_model=None, env={}, concurrency=1, + say=lambda s: None, job_dir=self.dir, ids=self.units) + + def edit(self, unit, change): + trial = next(t for t in os.listdir(self.dir) if t.startswith(unit + "__")) + path = os.path.join(self.dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + change(res) + with open(path, "w") as f: + json.dump(res, f) + + def test_a_clean_job_passes(self): + r = self.held() + self.assertTrue(r["ok"], r["problems"]) + self.assertEqual((r["passed"], r["value"], r["units"]), ("0/3", 0.0, 3)) + + def test_a_reward_without_a_score_a_score_below_0_and_nan_are_no_score(self): + for value in ({"reward": 1.0}, {"score": -100}, {"score": float("nan")}, {"score": True}, {"score": "0"}): + self.edit("u2", lambda res: res["verifier_result"].update(rewards=value)) + r = self.held() + self.assertFalse(r["ok"], value) + self.assertIn("1 unit(s) ended with no score from 0 to 1 and no error: read the job's trials", r["problems"], value) + self.assertEqual(r["value"], 0.0) + + def test_a_trial_another_run_wrote_is_refused(self): + self.edit("u1", lambda res: res["standard"].update(benchmark="other", control={"name": "oracle"}, plugged="oracle")) + self.edit("u3", lambda res: res["standard"].update(job="another-job")) + r = self.held() + self.assertFalse(r["ok"]) + said = " | ".join(r["problems"]) + for k in ("benchmark", "control", "who ran it", "job"): + self.assertIn(f"whose own record names another {k} than the job", said) + + LEAKY = '''from nullius.standard.jsonl import serve import os ANSWER = "the reference answer is forty-two" diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py new file mode 100644 index 0000000..17506a1 --- /dev/null +++ b/app/tests/test_standard_gated.py @@ -0,0 +1,134 @@ +"""A gated benchmark's failing endpoint is said by its status alone, down to the trial's result.json.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius.standard import definitions, install, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +CONTROLS = {"nop": {"source": "nullius", "from": "Nothing runs; the answer is empty."}} +WORDS = "the gated task's own sentence" +FAILING = '''from nullius.standard.jsonl import serve +def score(unit, dir, transcript, answer, **_): + raise RuntimeError("the LLM grader's endpoint answered 402, and the task was scored without it") +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, "score": score}) +''' + + +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's grader said: status 402: " + %r) +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' % WORDS +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Calls: + """The unit's relay as Run.unit reads it: each route's calls, the judge's ending on a refusal with words.""" + def __init__(self, side): + self.side, self.dir = side, os.devnull + + def calls(self, tag): + if tag.endswith("-" + self.side): + return [{"status": 200}, {"status": 402, "error": f"status 402: {WORDS}"}] + return [] + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Gated(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "failing") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(FAILING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def saved(self, side, **bench): + b = dict(self.b, **bench) + jobs = os.path.join(self.tmp, "jobs-" + side + str(bench)) + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", jobs_root=jobs, env={}, + say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_refused_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.saved(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said = self.saved("judge") + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn(WORDS, said["exception_message"]) + + def answered(self, side, gated): + """drive_upstream's answer failing on its judge's or simulated user's refusal.""" + here = os.path.join(self.tmp, "paper") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + b = dict(self.b, _here=here, setups=SETUPS, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"answer-{side}-{gated}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_answer_failing_on_its_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.answered(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_an_answer_failing_on_a_benchmark_not_gated_keeps_its_words(self): + said = self.answered("judge", gated=False) + self.assertEqual(said["exception_type"], "NonZeroAgentExitCodeError") + self.assertIn(WORDS, said["exception_message"]) + + def test_the_failure_alone(self): + calls = [{"status": 429, "error": WORDS}] + self.assertEqual(relay.failure(calls), ("RateLimitError", WORDS)) + exc, said = relay.failure(calls, words=False) + self.assertEqual(exc, "RateLimitError") + self.assertNotIn(WORDS, said) + self.assertEqual(relay.failure([{"status": 404, "error": "no such model"}], words=False)[0], "ModelNotFoundError") + + def test_gated_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) + self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", + definitions.problems(dict(self.b, gated="yes"), "benchmark")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_parts.py b/app/tests/test_standard_parts.py index ea875b8..318af12 100644 --- a/app/tests/test_standard_parts.py +++ b/app/tests/test_standard_parts.py @@ -35,6 +35,15 @@ def trial(self, name, f1s): exception=None, calls=[], trajectory=None) return d + def test_a_stopped_score_says_so_in_the_result(self): + for name, score in (("stopped", {"reward": 0.5, "stopped": "batch cap"}), ("whole", {"reward": 0.5})): + job.write_trial(os.path.join(self.job, name), job="job", bench=BENCH, other={"key": "original", "_kind": "upstream"}, + model="m", unit=name, t0=0, t1=1, produce=(0, 1), score=score, exception=None, calls=[], trajectory=None) + with open(os.path.join(self.job, "stopped", "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + with open(os.path.join(self.job, "whole", "result.json")) as f: + self.assertNotIn("stopped", json.load(f)["verifier_result"]) + def test_the_mean_is_over_every_question(self): self.trial("conv-a", [1.0] * 4) # 4 questions, all right self.trial("conv-b", [0.0] * 12) # 12 questions, all wrong diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 8319794..d6748e4 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -3,6 +3,7 @@ A driver or capability process answers one JSON line per request.""" from __future__ import annotations +import base64 import http.client import json import os @@ -34,6 +35,22 @@ b'{"type":"message","content":[{"type":"output_text","text":"hi"}]}],' b'"usage":{"input_tokens":20,"output_tokens":5,"input_tokens_details":{"cached_tokens":10}}}}\n\n') +RESP_FAILED = json.dumps({"object": "response", "id": "gen-1", "status": "failed", "output": [], "usage": None, + "error": {"code": "server_error", "message": "the request said x"}}).encode() +RESP_FAILED_SSE = (b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","output":[],' + b'"error":{"code":"server_error","message":"the request said x"}}}\n\n') +RESP_INCOMPLETE = json.dumps({"object": "response", "status": "incomplete", "output": [], + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 30, "output_tokens": 8}}).encode() + +HOLLOW = {"hollow": json.dumps({"id": "gen-1", "provider": "Minimax", "model": "m"}).encode(), + "hollow-error": json.dumps({"id": "gen-1", "error": {"code": 502, "message": "the request said x"}}).encode(), + "hollow-429": json.dumps({"id": "gen-1", "error": {"code": 429, "message": "the request said x"}}).encode(), + "hollow-402": json.dumps({"id": "gen-1", "error": {"code": 402, "message": "the request said x"}}).encode(), + "hollow-404": json.dumps({"id": "gen-1", "error": {"code": 404, "message": "the request said x"}}).encode(), + "count": json.dumps({"input_tokens": 12}).encode(), "no-choice": json.dumps({"choices": []}).encode(), + "not-json": b"the request said x"} + class Upstream(BaseHTTPRequestHandler): seen = [] @@ -51,6 +68,23 @@ def do_POST(self): self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") elif mode == "no-usage": self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") + elif mode in ("400", "413", "422"): + self.reply(int(mode), b'{"error":{"message":"could not process image"}}', "application/json") + elif mode in ("cut", "short"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header(*(("Transfer-Encoding", "chunked") if mode == "cut" else ("Content-Length", str(len(CHAT_SSE) + 9)))) + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(CHAT_SSE), CHAT_SSE) if mode == "cut" else CHAT_SSE) + elif mode in HOLLOW: + self.reply(200, HOLLOW[mode], "application/json", *[("X-Generation-Id", "gen-1")] * (mode == "not-json"), + ("Retry-After", "30")) + elif self.path.endswith("/responses") and mode == "failed": + self.reply(200, RESP_FAILED, "application/json") + elif self.path.endswith("/responses") and mode == "failed-sse": + self.reply(200, RESP_FAILED_SSE, "text/event-stream") + elif self.path.endswith("/responses") and mode == "incomplete": + self.reply(200, RESP_INCOMPLETE, "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -58,10 +92,11 @@ def do_POST(self): else: self.reply(200, json.dumps(CHAT).encode(), "application/json") - def reply(self, status, body, kind): + def reply(self, status, body, kind, *headers): try: self.send_response(status) - self.send_header("Content-Type", kind) + for header in (("Content-Type", kind),) + headers: + self.send_header(*header) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -90,6 +125,7 @@ def post(self, tag, path, body): with urllib.request.urlopen(req) as r: return r.status, r.read() except urllib.error.HTTPError as e: + self.retry_after = e.headers.get("Retry-After") return e.code, e.read() def test_plain_reply_passes_through_and_the_real_key_goes_out(self): @@ -120,6 +156,68 @@ def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): self.assertEqual(rec["reply"]["text"], "hello") self.assertEqual(rec["reply"]["usage"]["input"], 7) + def test_a_stream_cut_off_after_its_headers_is_a_failed_call(self): + for mode in ("cut", "short"): + self.post("u" + mode, "/chat/completions", {"model": "m", "stream": True, "mode": mode, "messages": []}) + rec = self.relay.calls("u" + mode)[-1] + self.assertEqual(rec["status"], 502) + self.assertIn("cut off", rec["error"]) + self.assertEqual(rec["reply"]["usage"]["input"], 7) + self.assertEqual(job._usage_total([rec])["cost"], rec["cost_usd"]) + + def test_a_refused_call_keeps_each_inline_image_by_its_type_wholeness_and_size_alone(self): + png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20 + b"IEND\xaeB`\x82\n").decode() + cut = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20).decode() + jpeg = base64.b64encode(b"\xff\xd8\xff" + b"y" * 9 + b"\xff\xd9tail").decode() + webp = base64.b64encode(b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"z" * 4 + b"\n").decode() + image = lambda kind, data: {"type": "image", "source": {"type": "base64", "media_type": kind, "data": data}} + content = [image("image/png", png), image("image/png", cut), image("image/gif", "not base64!"), + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg}"}}, + {"type": "input_image", "image_url": f"data:image/webp;base64,{webp}"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "grade"}] + sent = [{"type": "image/png", "is": "image/png", "bytes": len(png)}, {"type": "image/png", "is": None, "bytes": len(cut)}, + {"type": "image/gif", "is": None, "bytes": 11}, {"type": "image/jpeg", "is": "image/jpeg", "bytes": len(jpeg)}, + {"type": "image/webp", "is": "image/webp", "bytes": len(webp)}] + for mode in ("400", "413", "422", "429", None): + self.post("img", "/messages", {"model": "m", "mode": mode, "messages": [{"role": "user", "content": content}]}) + rec = self.relay.calls("img")[-1] + with self.subTest(mode): + self.assertEqual(rec.get("images"), sent if mode in ("400", "413", "422") else None) + self.assertFalse(any(data in json.dumps(rec) for data in (png, cut, jpeg, webp))) + + def test_a_200_to_a_model_call_with_no_reply_is_a_502_said_by_its_kind_alone(self): + for mode, path, code, said, kind in ( + ("hollow", "/messages", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/chat/completions", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/responses", 502, "no reply", "NetworkConnectionError"), + ("hollow-error", "/messages", 502, "an error 502", "NetworkConnectionError"), + ("hollow-429", "/messages", 429, "an error 429", "RateLimitError"), + ("hollow-404", "/completions", 502, "an error 404", "NetworkConnectionError"), + ("hollow-402", "/messages", 402, "an error 402", "ApiError"), + ("not-json", "/messages", 502, "no reply", "NetworkConnectionError")): + tag = f"uh-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + rec = self.relay.calls(tag)[-1] + self.assertEqual(self.retry_after, "30") + with self.subTest(mode=mode, path=path): + self.assertEqual((status, rec["status"]), (code, code)) + self.assertEqual(json.loads(body)["error"]["message"], f"the endpoint answered 200 with {said}") + self.assertEqual(rec["error"], f"HTTP {code}: the endpoint answered 200 with {said}") + self.assertEqual(rec["answered"], 200) + self.assertFalse({"reply", "cost_usd"} & set(rec)) + self.assertNotIn("the request said", json.dumps(rec)) + self.assertEqual(relay.failure([rec], agent=True)[0], kind) + self.assertEqual(self.post("uk-asked-stream", "/messages", {"model": "m", "mode": "hollow", "stream": True, + "messages": []}), (200, HOLLOW["hollow"])) + for mode, path in (("count", "/messages/count_tokens"), ("no-choice", "/chat/completions"), ("hollow", "/models"), + ("hollow", "/threads/t1/messages")): + tag = f"uk-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + with self.subTest(path): + self.assertEqual((status, body), (200, HOLLOW[mode])) + self.assertEqual(self.relay.calls(tag)[-1]["status"], 200) + self.assertNotIn("answered", self.relay.calls(tag)[-1]) + def test_streamed_responses_read_from_the_completed_event(self): status, body = self.post("u3", "/responses", {"model": "m", "stream": True, "input": "x"}) self.assertEqual(body, RESP_SSE) @@ -128,6 +226,25 @@ def test_streamed_responses_read_from_the_completed_event(self): self.assertEqual(rec["reply"]["tool_calls"][0]["name"], "shell") self.assertEqual(rec["reply"]["usage"]["cached"], 10) + def test_a_failed_response_with_no_usage_costs_nothing(self): + for tag, mode in (("uf1", "failed"), ("uf2", "failed-sse")): + status, _ = self.post(tag, "/responses", {"model": "m", "mode": mode, "input": "x", "stream": mode == "failed-sse"}) + self.assertEqual(status, 200) + rec = self.relay.calls(tag)[-1] + self.assertEqual((rec["reply"]["end"], rec["reply"]["error"]["code"]), ("failed", "server_error")) + self.assertEqual((rec["cost_usd"], rec["cost_from"]), (0.0, "failed")) + failed = {"status": 200, "cost_usd": 0.0, "cost_from": "failed"} + self.assertEqual(job._usage_total([failed, {"status": 200}])["cost"], None) + self.assertEqual({k: job._usage_total([failed, {"status": 200, "cost_usd": 0.5, "cost_from": "provider"}])[k] + for k in ("cost", "cost_from")}, {"cost": 0.5, "cost_from": "provider"}) + + def test_an_incomplete_response_is_priced_from_its_usage(self): + self.post("uf3", "/responses", {"model": "m", "mode": "incomplete", "input": "x"}) + rec = self.relay.calls("uf3")[-1] + self.assertTrue(rec["reply"]["end"].startswith("incomplete")) + self.assertEqual(rec["cost_from"], "estimate") + self.assertGreater(rec["cost_usd"], 0) + def test_calls_are_recorded_by_their_unit(self): self.post("ua", "/chat/completions", {"model": "m", "messages": []}) self.post("ub", "/chat/completions", {"model": "m", "messages": []}) @@ -230,6 +347,54 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_stream_cut_off_or_a_failed_response_with_no_usage_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + for tag, path, body in (("uc", "/chat/completions", {"model": "m", "stream": True, "mode": "cut", "messages": []}), + ("uf", "/responses", {"model": "m", "mode": "failed", "input": "x"})): + req = urllib.request.Request(r.url(tag) + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,), (failed,) = r.calls("uc"), r.calls("uf") + self.assertEqual((cut["status"], failed["cost_from"]), (502, "failed")) + self.assertIsNotNone(cut.get("cost_usd")) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], cut["booked_usd"] + failed["booked_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_stream_cut_off_that_already_cost_more_than_its_booking_counts_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "stream": True, "mode": "cut", "messages": []} + req = urllib.request.Request(r.url("uc") + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with mock.patch.object(ledger, "bound", lambda *a, **k: 1e-9): + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,) = r.calls("uc") + got = led.state() + self.assertEqual((cut["status"], got["unsettled_usd"]), (502, 0.0)) + self.assertGreater(cut["cost_usd"], cut["booked_usd"]) + self.assertAlmostEqual(got["spent_usd"], cut["cost_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_unit_asked_to_stop_books_nothing_and_a_200_with_no_reply_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + with open(os.path.join(r.dir, relay.STOP), "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(r, "u1", {"model": "m", "messages": []})[0], 400) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"], got["unsettled_usd"]), (0.0, 0, 0.0)) + os.unlink(os.path.join(r.dir, relay.STOP)) + self.assertEqual(self.post(r, "u1", {"model": "m", "mode": "hollow", "messages": []})[0], 502) + (call,) = r.calls("u1") + got = led.state() + self.assertEqual((call["answered"], got["spent_usd"], got["held_usd"]), (200, 0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], call["booked_usd"]) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply (2 output tokens here) a call that names no limit is booked those, at the same price.""" @@ -451,6 +616,11 @@ def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): self.assertEqual(relay._request_tail({"input": [], "reasoning": {"effort": "high"}})["reasoning_effort"], "high") self.assertIsNone(relay._request_tail({"messages": []})["reasoning_effort"]) + def test_reasoning_is_read_as_reasoning_or_thinking_tokens(self): + self.assertEqual(relay._usage({"completion_tokens_details": {"reasoning_tokens": 3}})["reasoning"], 3) + self.assertEqual(relay._usage({"output_tokens_details": {"reasoning_tokens": 5}})["reasoning"], 5) + self.assertEqual(relay._usage({"output_tokens": 9, "output_tokens_details": {"thinking_tokens": 7}})["reasoning"], 7) + class ServeTest(unittest.TestCase): """The relay as a process of its own: the key from its environment only, diff --git a/app/tests/test_standard_relay_pin.py b/app/tests/test_standard_relay_pin.py new file mode 100644 index 0000000..cfe525f --- /dev/null +++ b/app/tests/test_standard_relay_pin.py @@ -0,0 +1,263 @@ +"""A pinned route's failure on top of the serving pin (``serving``): a pinned call no endpoint took, or one its reply +says another provider served, fails the unit as its route's. Before a fake OpenRouter.""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import tempfile +import threading +import unittest +from unittest import mock +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from nullius.runs import classify +from nullius.standard import definitions, install, relay, run + +M3, SONNET = "minimax/minimax-m3", "anthropic/claude-sonnet-4.6" +PINS = {M3: ["minimax"], SONNET: ["anthropic"]} +SERVES = {M3: ["Parasail", "MiniMax"], SONNET: ["Google", "Anthropic"]} +SECRET = "the words of a gated task" +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") + + +class OpenRouter(BaseHTTPRequestHandler): + """Routes as OpenRouter documents them: ``down`` a 503, no endpoint of the pin a 404.""" + ignore_pin = False + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model, pref = body.get("model"), body.get("provider") or {} + allowed = [e for e in SERVES.get(model, ["Someone"]) + if OpenRouter.ignore_pin or not pref.get("order") or e.lower() in pref["order"]] + if body.get("down"): + allowed = [] + if not allowed: + status = 503 if body.get("down") else 404 + return self.reply(status, {"error": {"code": status, "message": f"No allowed providers are available for the selected model: {SECRET}"}}) + if self.path.endswith("/messages"): + return self.reply(200, {"id": "gen-555", "type": "message", "role": "assistant", "content": [{"type": "text", "text": SECRET}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) + self.reply(200, {"id": "gen-123", "provider": allowed[0], "choices": [{"message": {"role": "assistant", "content": SECRET}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": 0.0001}}) + + def reply(self, status, out): + body = json.dumps(out).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + try: + return e.code, e.read() + finally: + e.close() + + +class Pinned(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.relay = self.pinned(PINS) + + def pinned(self, serving, model=M3): + r = relay.Relay(self.base, "sk-real", tempfile.mkdtemp(dir=self.dir), model=model, + side_models=["claude-sonnet-4-6=" + SONNET], serving=serving).start() + self.addCleanup(r.stop) + return r + + def test_no_endpoint_of_the_pin_is_the_routes_error_not_an_empty_answer(self): + r = self.pinned({M3: ["fireworks"]}) + status, _ = post(r.url("u3") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertEqual(status, 404) + kind, said = relay.failure(r.calls("u3"), serving={M3: ["fireworks"]}) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to fireworks", said) + self.assertEqual(classify.from_verdict(kind, None), classify.API_ERROR) + + def test_a_pinned_provider_down_is_the_routes_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + (call,) = self.relay.calls("u1") + self.assertEqual(call["status"], 503) + self.assertEqual(relay.failure(self.relay.calls("u1"), serving=PINS)[0], "ApiProviderRouteError") + + def test_a_gated_benchmark_keeps_the_endpoints_words_out_of_a_route_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + kind, said = relay.failure(self.relay.calls("u1"), words=False, serving=PINS) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to minimax", said) + self.assertIn("answered 503", said) + self.assertNotIn(SECRET, said) + self.assertNotIn("No allowed providers", said) + + def test_an_unpinned_404_is_still_what_it_was(self): + calls = [{"status": 404, "model": "other/model", "error": "HTTP 404: model not found"}] + self.assertEqual(relay.failure(calls, serving=PINS)[0], "ModelNotFoundError") + self.assertIsNone(relay.route_failure(calls, PINS)) + + def test_a_side_models_refusal_fails_the_unit_even_when_the_agent_answered_last(self): + unit = self.relay.url("u1") + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + calls = self.relay.calls("u1") + self.assertIsNone(relay.failure(calls, serving=PINS)) + kind, said = relay.route_failure(calls, PINS, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn(SONNET, said) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": []}) + self.assertIsNone(relay.route_failure(self.relay.calls("u1"), PINS)) + + def test_a_routes_calls_are_read_in_the_order_they_began(self): + pin, refused = {M3: ["fireworks"]}, {"model": M3, "status": 404} + self.assertIsNone(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, dict(refused, t0=1)], pin)) + self.assertEqual(relay.route_failure([dict(refused, t0=2), {"model": M3, "status": 200, "t0": 1}], pin)[0], + "ApiProviderRouteError") + self.assertEqual(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, refused], pin)[0], "ApiProviderRouteError") + + def test_a_judges_answer_does_not_cover_the_simulated_users_refusal_of_the_same_model(self): + unit, judge = self.relay.url("u1"), self.relay.url("u1-judge", model=SONNET) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + post(judge + "/messages", {"model": SONNET, "messages": []}) + routes = [self.relay.calls("u1"), self.relay.calls("u1-judge"), []] + self.assertEqual(relay.routes_failure(routes, PINS, words=False)[0], "ApiProviderRouteError") + self.assertIsNone(relay.routes_failure([self.relay.calls("u1-judge")], PINS)) + + def test_only_a_route_with_no_endpoint_stops_the_run(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + self.assertFalse(relay.route_dead(relay.failure(self.relay.calls("u1"), serving=PINS))) + r = self.pinned({M3: ["fireworks"]}) + post(r.url("u5") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertTrue(relay.route_dead(relay.failure(r.calls("u5"), words=False, serving={M3: ["fireworks"]}))) + self.assertFalse(relay.route_dead(("ApiError", "404: x"))) + + def test_an_answer_from_outside_the_pin_fails_the_unit_as_its_route(self): + pin = {"openai/gpt-5-mini": ["minimax"]} + r = self.pinned(pin, model="openai/gpt-5-mini") + SERVES["openai/gpt-5-mini"] = ["Parasail"] + self.addCleanup(SERVES.pop, "openai/gpt-5-mini") + OpenRouter.ignore_pin = True + self.addCleanup(setattr, OpenRouter, "ignore_pin", False) + post(r.url("u4") + "/chat/completions", {"model": "x", "messages": []}) + (call,) = r.calls("u4") + self.assertEqual((call["status"], call["reply"]["served_by"]), (200, "Parasail")) + self.assertIsNone(relay.failure(r.calls("u4"), serving=pin)) + kind, said = relay.route_failure(r.calls("u4"), pin, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("served by Parasail", said) + self.assertNotIn(SECRET, said) + self.assertIsNone(relay.route_failure(r.calls("u4"), {})) + + def test_a_provider_is_off_its_pin_only_when_its_name_shares_nothing_with_the_pin(self): + self.assertIs(relay.held("MiniMax", ["minimax"]), True) + self.assertIs(relay.held("Anthropic", ["anthropic"]), True) + self.assertIs(relay.held("Google AI Studio", ["google-ai-studio"]), True) + self.assertIs(relay.held("Parasail", ["minimax"]), False) + self.assertIsNone(relay.held("Google", ["google-vertex"])) + self.assertIsNone(relay.route_failure([{"status": 200, "model": "m", "reply": {"served_by": "Google"}}], {"m": ["google-vertex"]})) + + def test_a_name_that_is_not_a_providers_name_fails_nothing_and_is_never_said(self): + odd = [{"status": 200, "model": "m", "reply": {"served_by": "Parasail\n" + SECRET}}] + self.assertIsNone(relay.route_failure(odd, {"m": ["minimax"]}, words=False)) + + +class Definition(unittest.TestCase): + def test_a_benchmark_without_pins_runs_as_before(self): + benches, _ = definitions.load("benchmark") + for b in benches: + with self.subTest(b["key"]): + self.assertEqual(run.pins_of(b, {"_kind": "agent"}, "openrouter", "m"), {}) + for which in (b.get("setups") or {}): + if which == "original": + other, _ = definitions.setup(b, which, "", "openrouter") + self.assertNotIn("pins", other or {}) + + def test_the_definitions_pin_holds_whatever_served_by_says(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + make = lambda **kw: run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", + env={"OPENROUTER_API_KEY": "k"}, unsandboxed=True, jobs_root=tmp, say=lambda s: None, **kw) + self.assertEqual(make().serving, {SONNET: ["anthropic"]}) + r = make(served_by={"m-1": ["prov-a"], SONNET: ["google-vertex"]}) + self.assertEqual((r.serving, r.route_pins), ({"m-1": ["prov-a"], SONNET: ["anthropic"]}, {SONNET: ["anthropic"]})) + + +class Wired(unittest.TestCase): + """A unit of a run whose judge the definition pins and whose own model the user pins (``--served-by``).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(self.tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + self.r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "k"}, + unsandboxed=True, jobs_root=self.tmp, say=lambda s: None, served_by={"m-1": ["prov-a"]}) + + def ended(self, routes): + rr = mock.Mock(mcp_listeners=[], unit_image="", dir=self.tmp) + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 0} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit"): + stack.enter_context(mock.patch.object(self.r, name)) + self.noted = stack.enter_context(mock.patch.object(self.r, "note_refusal")) + stack.enter_context(mock.patch.object(self.r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(self.r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(self.r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = self.r.unit(0, "good") + return (res.get("exception_info") or {}).get("exception_type") + + def test_only_a_route_the_definition_pins_fails_the_unit_as_its_route(self): + ok, no = {"model": "m-1", "status": 200}, {"status": 404, "error": "HTTP 404: no endpoint"} + self.assertNotIn(self.ended({"u0000": [dict(no, model="m-1")]}), (None, "ApiProviderRouteError")) + self.assertIsNone(self.ended({"u0000": [dict(ok, reply={"served_by": "Parasail"})]})) + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertTrue(self.noted.call_args[0][0][1].startswith("404: ")) + self.assertEqual(self.ended({"u0000": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [{"model": SONNET, "status": 200, "reply": {"served_by": "Parasail"}}]}), + "ApiProviderRouteError") + + def test_three_units_in_a_row_with_no_endpoint_on_a_pinned_route_stop_the_run(self): + self.r._halt, self.r._refused, self.r.halted = threading.Event(), [], None + for _ in range(3): + self.r.note_refusal(("ApiProviderRouteError", "404: no endpoint")) + self.assertTrue(self.r._halt.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec26..137b12c 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -624,6 +624,25 @@ def unit(*a): got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + real = r.unit + + def unit(*a): + res = real(*a) + if res and res["trial_name"].startswith("model__"): + with open(os.path.join(r.job_dir, res["trial_name"], ".work", "relay", relay.STOP), "w") as f: + f.write("batch cap\n") + return res + r.unit = unit + job_dir = r.start() + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index c160df5..fe861a2 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -191,7 +191,8 @@ def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): """An error that echoes the request's credentials, and a routing object holding a token: the record keeps neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" - Upstream.always = (200, "application/json", json.dumps({"error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + Upstream.always = (200, "application/json", json.dumps({"choices": [{"message": {"content": "ok"}}], + "error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) for serving in (None, {"m-run": ["prov-a"]}): with self.subTest(serving=serving): r = self.relay(model="m-run", key=KEY, serving=serving) @@ -260,15 +261,15 @@ def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) - def test_what_the_record_read_before_is_read_as_before(self): - """The facts are read apart: a failed Responses stream's text, tool calls and usage stay as the relay read - them before (it took nothing from a ``response.failed`` event), its error beside them.""" + def test_a_failed_responses_stream_is_read_beside_its_facts(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage are read from its + ``response.failed`` event, its error beside them.""" body = sse({"type": "response.output_text.delta", "delta": "part"}, {"type": "response.failed", "response": {"object": "response", "status": "failed", "error": {"code": "server_error", "message": "failed"}, "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], "usage": {"input_tokens": 5, "output_tokens": 2}}}) - self.assertEqual(relay.read_reply("/responses", body, True), {"text": "", "tool_calls": [], "usage": None}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "x", "tool_calls": [], "usage": relay._usage({"input_tokens": 5, "output_tokens": 2})}) self.assertEqual(relay.reply_facts("/responses", body, True), {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) @@ -276,9 +277,7 @@ def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own is added, and a unit's failure reads it as before (a 200 is no failure).""" r = self.relay(model="m-run") - errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", - {"code": 502, "message": "Provider returned error"}), - (sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + errors = [(sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", {"code": "server_error", "message": "upstream ended"}), diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index 7d84a04..751625c 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -15,7 +15,7 @@ from types import SimpleNamespace from nullius.runs import launches, parse -from nullius.standard import job, merge, shard +from nullius.standard import job, merge, prove, shard BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} @@ -61,14 +61,15 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local"): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", bench_options=None): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) os.makedirs(d) planned = list(units) if planned is None else planned job.write_config(d, job=name, bench=bench, other=other, model=model, provider="openai", dataset="all", units=planned, - concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend) + concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend, + bench_options=bench_options) results = [] for k, u in enumerate(units): place = int(u.split("-")[1]) @@ -80,7 +81,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": 1.0 if place % 2 else 0.0}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) + job.write_result(d, name, t0, t0 + 10 * (min(index, 9) + 1), results, len(planned), budget=budget) return d @@ -199,6 +200,36 @@ def test_units_no_shard_ran_are_said(self): j = parse.parse_job(got["out"]) self.assertEqual((j.n_total_trials, len(j.trials)), (len(self.split[1]) + len(planned), len(self.split[1]) + len(planned) - 1)) + def test_a_launch_of_one_unit_per_shard_merges_and_says_only_the_units_it_lacks(self): + n = 2 ** 64 + units = [f"t-{i}" for i in range(10)] + dirs = [write_shard(self.tmp, f"one-{u}", [u], shard.of(u, n), n) for u in units[:-1]] + got = merge.merge(dirs, os.path.join(self.tmp, "out")) + self.assertEqual((got["trials"], got["notes"]), (9, ["1 of the dataset's 10 units are not in the merge"])) + + def test_shards_run_with_different_benchmark_options_keep_each_its_own(self): + same = {"websites": "none", "gitlab": "none"} + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, bench_options=same), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example")), + write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)] + std = read(os.path.join(merge.merge(dirs, os.path.join(self.tmp, "out"))["out"], "config.json"))["standard"] + self.assertNotIn("benchmark_options", std) + self.assertEqual([m["benchmark_options"]["gitlab"] for m in std["merged_from"]], ["none", "gitlab.example", "none"]) + alike = [write_shard(self.tmp, f"a{i}", self.split[i], i, 3, bench_options=same) for i in (1, 2, 3)] + std = read(os.path.join(merge.merge(alike, os.path.join(self.tmp, "alike"))["out"], "config.json"))["standard"] + self.assertEqual(std["benchmark_options"], same) + self.assertNotIn("benchmark_options", std["merged_from"][0]) + + def test_a_merge_of_merges_keeps_the_options_of_its_inner_shards(self): + same = {"websites": "none", "gitlab": "none"} + inner = merge.merge([write_shard(self.tmp, "s1", self.split[1], 1, 3), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example"))], + os.path.join(self.tmp, "inner"))["out"] + outer = merge.merge([inner, write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)], + os.path.join(self.tmp, "outer"), force=True)["out"] + self.assertEqual(prove._options_of(read(os.path.join(outer, "config.json"))), + [dict(same, gitlab="gitlab.example"), same, {}]) + def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): dirs = self.shards() os.makedirs(os.path.join(self.tmp, "taken", "x")) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267..ea68afc 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -6,6 +6,7 @@ import os import tempfile import unittest +from unittest import mock from nullius.standard import job from nullius.traces import trace @@ -139,6 +140,31 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) + def test_a_trace_that_fails_leaves_the_result_and_says_its_error(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + with mock.patch.object(job, "trace_from_run", side_effect=KeyError("the task's own words")): + path = self.write(AGENT, traj) + agent = os.path.dirname(path) + self.assertFalse(os.path.exists(path)) + with open(os.path.join(os.path.dirname(agent), "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["rewards"], {"reward": 1.0}) + with open(os.path.join(agent, "trace.error.txt")) as f: + self.assertEqual(f.read(), "KeyError\n") + self.assertTrue(os.path.exists(os.path.join(agent, "trajectory.json"))) + + def test_a_trace_error_that_cannot_be_written_leaves_the_result(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + write = job._write + + def refusing(path, text): + if os.path.basename(path).startswith("trace"): + raise PermissionError(path) + write(path, text) + + with mock.patch.object(job, "_write", refusing): + path = self.write(AGENT, traj) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(os.path.dirname(path)), "result.json"))) + class Backfill(unittest.TestCase): """A trial written before traces gets one from the files it kept, and nothing else changes.""" diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index 6ed5e16..594fd76 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -115,7 +115,7 @@ def tree(job_dir): mock.patch.object(install, "upstream", side_effect=install.InstallFailed("git fetch: exit 128")), \ self.assertRaises(run.Refused): preparing.start() - self.assertEqual(tree(preparing.job_dir), [".work", "progress.jsonl"]) + self.assertEqual(tree(preparing.job_dir), [".work", ".work/job.lck", "progress.jsonl"]) self.assertEqual(tree(unknown.job_dir), tree(preparing.job_dir)) self.assertIn("nope", progress.refusal(progress.read(unknown.job_dir))) diff --git a/app/tests/test_standard_versions.py b/app/tests/test_standard_versions.py index 898a0dc..1c86087 100644 --- a/app/tests/test_standard_versions.py +++ b/app/tests/test_standard_versions.py @@ -217,7 +217,7 @@ def test_new_version_copies_the_default_and_refuses_a_duplicate(self): ''' TOY = dict(BENCH, upstream={"license": "MIT"}, default="1", controls={"nop": {"source": "nullius", "from": "nothing runs"}}, - versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 2}], + versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 3}], "driver": {"command": ["python3", "{here}/driver2.py"]}, "proven": {}}, "1": {"proven": {"all": {"units": {"units": 2, "passed": "2 ids", "on": "2026-10-12"}}}}}) diff --git a/docs/cli/read-jobs.md b/docs/cli/read-jobs.md index 86ca991..c690a67 100644 --- a/docs/cli/read-jobs.md +++ b/docs/cli/read-jobs.md @@ -56,7 +56,7 @@ nullius standard traces [--dry-run] ## Give past standard runs a trace -A standard run writes `agent/trace.json` as it ends. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. +A standard run writes `agent/trace.json` as it ends; one it cannot build leaves `agent/trace.error.txt`, the error's type, and the trial's result as it is. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. ```bash title="See what it would write, then write it" nullius standard traces ./jobs --dry-run diff --git a/docs/cli/run.md b/docs/cli/run.md index ce99f7f..95e2112 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -101,11 +101,11 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | Rule | Detail | | --- | --- | | A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | -| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | +| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings. A model the benchmark pins for the run ([standard](../standard.md), Pinned routes) keeps the benchmark's providers | | What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`). A Responses reply names the provider that served it there alone. A Messages request is sent without the header, since its reply names its provider. A model with no pin is sent as before | | Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | -| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does, but for a model the benchmark pins ([standard](../standard.md), A refused route) | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record, but for a failed Responses object's cost, and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a model the benchmark pins and a 200 with no reply ([standard](../standard.md)) | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | | What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name. Where that name is not the slug once lowercased, `held` reads `unknown` though the provider served the call (`Google` for `google-vertex`, `SambaNova` for `sambanova-turbo`) | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | diff --git a/docs/standard.md b/docs/standard.md index 416feb7..28aa023 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -141,16 +141,18 @@ record); verifier/reward.txt and score.json | Rule | Detail | | --- | --- | | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | -| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | +| A relay per unit | Its record is the trial's `.work/relay` (an earlier run's under the same job name is moved aside to `.work/earlier.`, with its `result.json`, `verifier/` and `agent/`; a run holds its job name until its process ends, and one under a name another still holds is refused before it writes anything); the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A model call answered 200 with no reply in it, not streamed, reaches its caller and the record as a 502, or the 429 or 402 of an error it held, that says only its kind; the record keeps `answered: 200`. A call refused with 400, 413 or 422 keeps, as `images`, each inline image it sent by its declared type, the type its bytes show whole and its size, never a byte | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | -| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a route the definition pins (A refused route) and a 200 with no reply (A relay per unit) | | Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | +| Pinned routes | On OpenRouter, a model the definition pins (`setups.original.pin` in a run of the Original, `score.judge.pin` while the official judge grades the run) is pinned as `--served-by` pins it, whatever `--served-by` says of that model. The pin follows the model sent, so a model the agent shares with the judge goes to the judge's providers | +| A refused route | A call the definition pins (Pinned routes) answered 404 or 503 is refused. When a model's last call on a route was refused (each route read on its own), or any such call's `reply.served_by` shares nothing with the pin's slugs (a name that is not its slug, `Google` for `google-vertex`, fails no call on a guess), the unit fails with `ApiProviderRouteError` unless its agent's own last call failed it first, an API error outside the agent's rate, even when its code made nothing of it. A gated benchmark keeps it to the status, the pin and provider names. Three units in a row whose route has no endpoint (404) stop the run, as a refused key does | ## Safety @@ -345,6 +347,8 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model). With `--units`, `units_total` counts the listed units. The shards' lists hold the same units; a list's file name, comments and sha256 may differ from shard to shard | | `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, serving providers, backend or N are refused unless `--force`. Shards whose lists hold different units, and two trials of one unit, are refused always. `merge` compares the units each list holds, in any order; the sha256 is kept in the record only | | `cli prove --shard I/N` | The controls and the leak check on that shard's units; each record says `shard` | +| Different options per shard | Shards of one launch may differ in `--bench-option` (a unit that needs a service and a write share of its own); the merged config keeps each shard's in `merged_from` | +| `cli prove --job nop=MERGED` | Holds a merged launch of a control to its bound instead of running it. The job and every trial's own record must name that benchmark, version and control and this job; each unit of the dataset exactly once, with the field its score kind names (a number from 0 to 1) or an error; and the units check must pass. `--dataset` named, no `--limit` or `--shard`. The record names the job and the options it ran with. The runs pages group a launch's shards by their shared `standard.launch` (`--launch`); a name like `______prove-nop` is only a convention for reading | ```bash title="Two machines, one merge" nullius standard run humaneval oracle --shard 1/2 --limit 6 # machine 1 @@ -412,15 +416,19 @@ use the source folders and repository checks. | `turn` | The transcript so far; `simulator: {model, base_url, key}` when there is one; `judge` for an upstream that grades as the conversation ends; `attempt` with `limits.attempts` | The next message, `{done: true}`, or `restart` | | `produce` | A capability benchmark's recall, `base_url` and `key` | The reading, through the relay | | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | -| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | +| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by`; `stopped`, the reason of a stop asked while the unit ran, kept as `verifier_result.stopped` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | | Field in `standard.json` | Meaning | | --- | --- | -| `score.judge` | `{model, says}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another | +| `gated` | `true` for a benchmark whose tasks are gated: a unit's failing model endpoint (the agent's, the judge's, the simulated user's) is kept as its status and class, never the provider's words, which can quote the request | +| `score.judge` | `{model, says, pin?}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another. `pin` (`{"openrouter": [provider slug]}`) pins every call sent as the official judge's model while the official judge grades the run, an agent's own included; another judge is pinned only when it is a model the setup pins | | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | +| `limits.turn_seconds` | How long the agent's turn may take (default 900); a unit's `agent_seconds` or a setup's own `turn_seconds` replaces it | | `limits.turns` | At most this many agent turns in a conversation (default 20) | +| `limits.score_seconds` | How long `score` may take (default 600). A driver that runs the unit's environment in `score`, as its null control does, sets it | +| `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -584,9 +592,11 @@ already accepts arbitrary models needs only the `original` method. | Rule | Detail | | --- | --- | | `original` driver method | A model of `models` only, by the name the upstream knows; the relay sends it as `served_as` says, with the prefix `providers` gives. A provider not in `providers` is refused. A model the upstream asks besides the answerer is a `side_models` entry. Without `models`, any model: its own agent runs as the run's model | -| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError` | +| `pin` | `{"openrouter": {model: [provider slug]}}`, by the names in `models` and `side_models`: on OpenRouter each goes to those providers only, with no fallback, so the original runs on the paper's endpoint. Other setups and agents are not pinned, but for a call sent as the official judge's model while it grades the run (`score.judge.pin`) | +| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError`, when its last reply (by when it was asked) has neither text nor a tool call; a last Responses reply answered 200 and `failed` is an API error. A runner that fails once a guard's `stop` beside the relay's records asks the unit to stop is `CancelledError` | | `units` | A setup may count its own units; `units` is sent `setup` | | `answer` | Runs the upstream's answerer through the unit's relay; `score` is sent `setup` too. Local processes or the Original setup driver container | +| `comparable` | `false` when no model of `models` is the paper's, so its runs compare with each other only; or the list of those models whose runs compare with the paper's (`paper`), when the setup runs several agents and only some are the paper's: the builder offers those alone as Like the paper, the others as Any of your choice | | Recorded | `standard.setup` in the config and each trial: its name, the model `asked` and `sent`, `comparable_with_paper` | | Attempts | `cli run --repeat K` runs each unit K times; `result.json` carries `pass_hat_k` (C(c, k) / C(n, k) averaged over the units, an errored unit left out) | @@ -629,7 +639,7 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c `cli run KEY nop` and `cli run KEY oracle` need no `--model` and no `--unsandboxed`: no model is called but a judged benchmark's judge. The config and every trial record `standard.control`. -`cli prove KEY [--dataset D] [--limit N]` runs, for each dataset: +`cli prove KEY [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop=JOB_DIR]` runs, for each dataset: | Check | Passes when | | --- | --- | @@ -638,6 +648,8 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c | `units` | Two `units` calls give the same ids: unique, plain enough to name a trial, as many as the dataset says | | `leak` | After `setup`, the oracle's answer and the lines it writes are not already in the prompt or the workspace. A reference under 12 characters is not looked for; the record says how many were skipped | +`--bench-option NAME=VALUE` sets the benchmark's own options for the units check's driver and each control's run, resolved and refused as `cli run` does them, the programs a value needs on this machine included, before anything is installed or started; the control job's config records the values. + What passed is written into `standard.json` per dataset, dated; a failing check is taken back, and a run on fewer units does not replace one on more. ```json title="standard.json ยท proven" From bef279f16f577593f58c6b1bba0ebf3c9157bf27 Mon Sep 17 00:00:00 2001 From: Oleg <247799477+sirotaoleg@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:10:40 +0000 Subject: [PATCH 23/24] feat(standard): let a benchmark run no rate-limited unit again, and keep the score of a unit a guard stopped despite a last 429 limits.run_again false: a unit that ended rate-limited is not run once more; its outcome stands. A unit whose driver scored it after a guard's stop keeps that score, marked stopped, though its last own call got a 429. --- app/nullius/standard/definitions.py | 2 ++ app/nullius/standard/run.py | 5 +++++ app/tests/test_standard_context_window.py | 21 +++++++++++++++++---- app/tests/test_standard_controls.py | 5 +++++ app/tests/test_standard_run.py | 10 ++++++++++ docs/standard.md | 3 ++- 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 0a6e2b1..b6e27b5 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -1366,6 +1366,8 @@ def problems(d: dict, kind: str) -> List[str]: v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") + if "run_again" in lim and not isinstance(lim["run_again"], bool): + bad.append("limits.run_again: false when a rate-limited unit is not run once more") a = lim.get("attempts") if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index bd9d9c6..5a4da6b 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -902,6 +902,8 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, the trial's own name and counted (into ``done``, as each ends). Nothing is run again once the run is stopped or halted, nor a unit a guard's stop asked to stop; the spending cap goes on counting every call.""" + if (self.bench.get("limits") or {}).get("run_again") is False: + return by_name = {self.trial_name(*iur): iur for iur in work} todo = [by_name[r["trial_name"]] for r in results if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name @@ -1167,6 +1169,9 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") or self.upstream and exc and ": VerifierError: " in exc[1]): failed = None + if failed and failed[0] == pacing.RATE_LIMITED and (score or {}).get("stopped") and rl is not None \ + and relay.asked(rl.dir) is not None: + failed = None if not failed and rl is not None and getattr(self, "route_pins", None): failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py index 66c6d94..d17ca2d 100644 --- a/app/tests/test_standard_context_window.py +++ b/app/tests/test_standard_context_window.py @@ -115,8 +115,8 @@ def test_it_is_the_agents_result_not_the_infrastructure(self): class Calls: """The agent's calls ending on an overflow, then a refused side call.""" - def __init__(self, own): - self.own, self.dir = own, os.devnull + def __init__(self, own, dir=os.devnull): + self.own, self.dir = own, dir def calls(self, tag): return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) @@ -144,12 +144,12 @@ def tearDown(self): install.HOME = self._home shutil.rmtree(self.tmp, ignore_errors=True) - def ran(self, own, gated=True): + def ran(self, own, gated=True, record=os.devnull): b = dict(self.b, **({"gated": True} if gated else {})) o, why = definitions.setup(b, "original", "paper-1", "custom") self.assertIsNotNone(o, why) jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") - with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own)): + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own, record)): job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) name = next(t for t in os.listdir(job_dir) if "__u" in t) @@ -180,6 +180,19 @@ def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) self.assertEqual(said["exception_type"], "RateLimitError") + def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_a_429(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace('"stopped": "context window"', '"stopped": "batch cap"')) + record = os.path.join(self.tmp, "record") + os.makedirs(record) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("batch cap\n") + said, ran = self.ran([AGENT_OK, {"status": 429, "error": "slow down"}], record=record) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: f.write(UNREAD) diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index 7bd8982..798d93a 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -226,6 +226,11 @@ def test_the_limit_is_a_whole_number_of_seconds_from_1(self): for v in (0, -1, 1.5, "600", True, 100001): self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + def test_run_again_is_true_or_false(self): + self.assertEqual(self.problems({"run_again": False}), []) + for v in (0, "no"): + self.assertIn("limits.run_again: false when a rate-limited unit is not run once more", self.problems({"run_again": v}), v) + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) self.assertEqual(r.score_seconds, 600) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 137b12c..4defc5c 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -624,6 +624,16 @@ def unit(*a): got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_benchmark_that_says_so_runs_no_rate_limited_unit_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}], + limits={"run_again": False}) + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): Echo.status = 429 b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) diff --git a/docs/standard.md b/docs/standard.md index 28aa023..6e230ae 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -144,7 +144,7 @@ record); verifier/reward.txt and score.json | A relay per unit | Its record is the trial's `.work/relay` (an earlier run's under the same job name is moved aside to `.work/earlier.`, with its `result.json`, `verifier/` and `agent/`; a run holds its job name until its process ends, and one under a name another still holds is refused before it writes anything); the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A model call answered 200 with no reply in it, not streamed, reaches its caller and the record as a 502, or the 429 or 402 of an error it held, that says only its kind; the record keeps `answered: 200`. A call refused with 400, 413 or 422 keeps, as `images`, each inline image it sent by its declared type, the type its bytes show whole and its size, never a byte | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a unit a guard's stop ended keeps the score its driver gave, a last 429 notwithstanding; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | @@ -429,6 +429,7 @@ use the source folders and repository checks. | `limits.turns` | At most this many agent turns in a conversation (default 20) | | `limits.score_seconds` | How long `score` may take (default 600). A driver that runs the unit's environment in `score`, as its null control does, sets it | | `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | +| `limits.run_again` | `false`: a unit that ended rate-limited is not run once more after the others; its outcome stands | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | From 369054a84fbc508d6572bff83d579c8107d47cd9 Mon Sep 17 00:00:00 2001 From: Oleg <247799477+sirotaoleg@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:10:40 +0000 Subject: [PATCH 24/24] fix(standard): keep a guard-stopped unit's score only for the agent's own last 429, not a side call's or a context window's stop The rule reads the agent's own last call, a guard's line in the stop file and a score stopped by other than the context window; a cap's refusal is still the cap's. limits.run_again takes null as unset. The rate limits row names the opt-out. --- app/nullius/standard/definitions.py | 2 +- app/nullius/standard/run.py | 5 +++-- app/tests/test_standard_context_window.py | 26 +++++++++++++++++++---- app/tests/test_standard_controls.py | 3 ++- docs/standard.md | 4 ++-- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index b6e27b5..20b4f1f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -1366,7 +1366,7 @@ def problems(d: dict, kind: str) -> List[str]: v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") - if "run_again" in lim and not isinstance(lim["run_again"], bool): + if lim.get("run_again") is not None and not isinstance(lim["run_again"], bool): bad.append("limits.run_again: false when a rate-limited unit is not run once more") a = lim.get("attempts") if a is not None and (not isinstance(a, str) or not a.strip()): diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 5a4da6b..ce5854a 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -1169,8 +1169,9 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") or self.upstream and exc and ": VerifierError: " in exc[1]): failed = None - if failed and failed[0] == pacing.RATE_LIMITED and (score or {}).get("stopped") and rl is not None \ - and relay.asked(rl.dir) is not None: + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and (score or {}).get("stopped") not in (None, "", relay.OVERFLOW) \ + and rl is not None and relay.asked(rl.dir) is not None \ + and [c.get("status") for c in relay._begun(calls) if not c.get("side")][-1:] == [429]: failed = None if not failed and rl is not None and getattr(self, "route_pins", None): failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py index d17ca2d..49fcc16 100644 --- a/app/tests/test_standard_context_window.py +++ b/app/tests/test_standard_context_window.py @@ -187,11 +187,29 @@ def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_ os.makedirs(record) with open(os.path.join(record, relay.STOP), "w") as f: f.write("batch cap\n") - said, ran = self.ran([AGENT_OK, {"status": 429, "error": "slow down"}], record=record) - self.assertIsNone(said) - self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) - said, _ = self.ran([AGENT_OK, {"status": 429, "error": "slow down"}]) + limited = {"status": 429, "error": "slow down"} + for calls in ([AGENT_OK, limited], [AGENT_OK, limited, {"status": 502, "error": "down", "side": True}]): + said, ran = self.ran(calls, record=record) + self.assertIsNone(said, calls) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + with open(os.path.join(ran.trials[0].dir, "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + said, _ = self.ran([AGENT_OK, limited]) self.assertEqual(said["exception_type"], "RateLimitError") + for own in (AGENT_OK, {"status": 401, "error": "no"}, {"status": 502, "error": "down"}): + said, _ = self.ran([own, dict(limited, side=True)], record=record) + self.assertEqual(said["exception_type"], "RateLimitError", own) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("context window\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + with open(os.path.join(record, relay.STOP), "a") as f: + f.write("batch cap\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace(', "stopped": "context window"', "")) + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index 798d93a..7dd4f93 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -227,7 +227,8 @@ def test_the_limit_is_a_whole_number_of_seconds_from_1(self): self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) def test_run_again_is_true_or_false(self): - self.assertEqual(self.problems({"run_again": False}), []) + for v in (False, True, None): + self.assertEqual(self.problems({"run_again": v}), [], v) for v in (0, "no"): self.assertIn("limits.run_again: false when a rate-limited unit is not run once more", self.problems({"run_again": v}), v) diff --git a/docs/standard.md b/docs/standard.md index 6e230ae..8097f5b 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -144,8 +144,8 @@ record); verifier/reward.txt and score.json | A relay per unit | Its record is the trial's `.work/relay` (an earlier run's under the same job name is moved aside to `.work/earlier.`, with its `result.json`, `verifier/` and `agent/`; a run holds its job name until its process ends, and one under a name another still holds is refused before it writes anything); the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A model call answered 200 with no reply in it, not streamed, reaches its caller and the record as a 502, or the 429 or 402 of an error it held, that says only its kind; the record keeps `answered: 200`. A call refused with 400, 413 or 422 keeps, as `images`, each inline image it sent by its declared type, the type its bytes show whole and its size, never a byte | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a unit a guard's stop ended keeps the score its driver gave, a last 429 notwithstanding; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | -| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a unit a guard's stop ended keeps the score its driver gave, its own last call's 429 notwithstanding; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | +| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Unless the benchmark's `limits.run_again` is `false`, each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. Provider `anthropic` speaks Anthropic's Messages (`ANTHROPIC_API_KEY` as `x-api-key`) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | | Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a route the definition pins (A refused route) and a 200 with no reply (A relay per unit) |