diff --git a/AGENTS.md b/AGENTS.md index 8a336c0..aa141f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,7 +67,11 @@ and its resources. The source layout stays under `app/`. tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: a benchmark's `standard.json` and `driver` in its folder (τ³-bench's also the MCP server of its domains' tools, `mcp_server`: the upstream's own, run on the machine; OSWorld1's also its null control's - agent, `nop`), `catalogue/agents//` + agent, `nop`; OSWorld2's also `nop`, `api_logs` (each image of its API logs kept once), `websites` (its mock + websites: the checks before a VM start, Overleaf's lock and reset, the stack's command line), `gitlab_server` (its + GitLab: the check before a VM start, its tasks' lock, the stack's command line), `proxy` (a + tinyproxy of a unit's own for the tasks that ask for a proxy) and, in `startup/`, + `sitecustomize`: its VMs' disk write limit in every process its driver starts), `catalogue/agents//` (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), diff --git a/app/nullius/catalogue/benchmarks/osworld2/agents.json b/app/nullius/catalogue/benchmarks/osworld2/agents.json new file mode 100644 index 0000000..dcbb2d4 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/agents.json @@ -0,0 +1,83 @@ +{ + "about": "The upstream's agents the driver runs, by the agent option: each its own script at the tested commit, as published, on one task. argv is what its scripts/bash wrapper passes (run_multienv_m3.sh, run_multienv_muse_spark.sh), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the image's password as the wrapper passes it, the task's meta file, the unit's results) and less the aws provider's --region and --use_public_ip and the wrapper's one --specific_task_id. M3 also gets the relay's key as --api_key, since its script reads MINIMAX_ANTHROPIC_BASE_URL and MINIMAX_API_KEY before ANTHROPIC_*, and a .env file in the upstream (each script calls load_dotenv) could set them. Muse Spark gets the reasoning effort and output budget of its published run's args.json (max, 262144) in place of its wrapper's (high, 8192), and MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT, since its count of input tokens before each call asks the endpoint for a path OpenRouter does not serve. env points each script's model client at the unit's relay too, and its simulated user's, asked by the name the documented invocation gives it (OSWORLD_USER_SIM_MODEL), which the Original setup sends on as a side model. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1 (the Anthropic client adds /v1/messages), {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples, {max_steps} the max_steps option's value. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them.", + "common": [ + "--provider_name", + "docker", + "--path_to_vm", + "{vm}", + "--headless", + "--num_envs", + "1", + "--test_config_base_dir", + "{examples}", + "--test_all_meta_path", + "{meta}", + "--result_dir", + "{results}", + "--client_password", + "osworld-public-evaluation" + ], + "agents": { + "m3": { + "script": "scripts/python/run_multienv_m3.py", + "models": [ + "MiniMax-M3" + ], + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "8192", + "--max_steps", + "{max_steps}", + "--model", + "{model}", + "--base_url", + "{base}", + "--api_key", + "{key}" + ], + "env": { + "ANTHROPIC_BASE_URL": "{base}", + "ANTHROPIC_API_KEY": "{key}", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + }, + "muse_spark": { + "script": "scripts/python/run_multienv_muse_spark.py", + "models": [ + "muse-spark-1.3" + ], + "argv": [ + "--action_space", + "pyautogui", + "--observation_type", + "screenshot", + "--max_tokens", + "262144", + "--reasoning_effort", + "max", + "--max_steps", + "{max_steps}", + "--model", + "{model}", + "--base_url", + "{base_url}", + "--save_model_eval_raw_info" + ], + "env": { + "MUSE_SPARK_API_KEY": "{key}", + "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1", + "OSWORLD_USER_SIM_PROVIDER": "anthropic", + "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "{base}", + "OSWORLD_USER_SIM_API_KEY": "{key}" + } + } + } +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/api_logs.py b/app/nullius/catalogue/benchmarks/osworld2/api_logs.py new file mode 100644 index 0000000..7faca44 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/api_logs.py @@ -0,0 +1,232 @@ +"""OSWorld2's api_logs with each image in them kept once. + +The upstream's M3 agent logs each model call's request with its screenshots as +base64, so one image repeats across files. pack(folder) writes each image once +as images/. and puts that path in its place; restore(folder) gives +each file back byte for byte. Each write is a temporary file renamed over its +name, so a failure leaves every file whole and running again changes nothing. + +python3 api_logs.py pack|restore FOLDER... prints what it did, as JSON.""" +import base64 +import binascii +import contextlib +import fcntl +import hashlib +import json +import os +import re +import stat +import sys +import tempfile + +IMAGES = "images" +TEMPORARY = ".nullius-tmp" +SHORTEST = 256 +VALUE = rb'(?:(?<=: )|(?<=\[)|(?<=, ))"' +BASE64 = re.compile(VALUE + rb'([A-Za-z0-9+/]{%d,}={0,2})"' % SHORTEST) +MARK = b'{"_packed_images": "' + IMAGES.encode() + b'", ' +LINK = re.compile(VALUE + IMAGES.encode() + rb'/([0-9a-f]{64}\.(?:jpg|png|gif|webp))"') + + +def _kind(data): + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "webp" + for magic, ext in ((b"\xff\xd8\xff", "jpg"), (b"\x89PNG\r\n\x1a\n", "png"), (b"GIF87a", "gif"), (b"GIF89a", "gif")): + if data.startswith(magic): + return ext + return None + + +def _image(text): + """The image a base64 value holds, as (its file name, its bytes), or None.""" + if len(text) % 4: + return None + try: + data = base64.b64decode(text, validate=True) + except (binascii.Error, ValueError): + return None + ext = _kind(data) + if ext is None or base64.b64encode(data) != text: + return None + return f"{hashlib.sha256(data).hexdigest()}.{ext}", data + + +def _expand(text, image): + """A packed file's text as it was, with ``image`` giving a file name's bytes.""" + return b"{" + LINK.sub(lambda m: b'"' + base64.b64encode(image(m.group(1).decode())) + b'"', text[len(MARK):]) + + +def _read(path, folder=None): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=folder) + with os.fdopen(fd, "rb") as f: + st = os.fstat(f.fileno()) + if not stat.S_ISREG(st.st_mode): + raise OSError(f"{path} is not a plain file") + return f.read(), st + + +def _write(path, data, like): + fd, tmp = tempfile.mkstemp(prefix="." + os.path.basename(path) + ".", suffix=TEMPORARY, dir=os.path.dirname(path)) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + f.flush() + os.fchmod(f.fileno(), stat.S_IMODE(like.st_mode)) + os.fsync(f.fileno()) + os.utime(tmp, ns=(like.st_atime_ns, like.st_mtime_ns)) + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + + +def _sync(folder): + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _held(path, name, folder=None): + """The bytes of ``path``, a plain file holding the image ``name`` names; None for anything else.""" + try: + data = _read(path, folder)[0] + except OSError: + return None + return data if f"{hashlib.sha256(data).hexdigest()}.{_kind(data)}" == name else None + + +def _inner(fd, name): + return os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + + +def _tidy(fd): + for name in os.listdir(fd): + if name.startswith(".") and name.endswith(TEMPORARY): + with contextlib.suppress(OSError): + os.unlink(name, dir_fd=fd) + + +@contextlib.contextmanager +def _alone(folder): + """The folder locked for this process, cleared of a stopped one's half-written files; None when another holds it.""" + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + yield None + return + _tidy(fd) + with contextlib.suppress(OSError): + images = _inner(fd, IMAGES) + try: + _tidy(images) + finally: + os.close(images) + yield fd + finally: + os.close(fd) + + +def _files(folder): + for name in sorted(os.listdir(folder)): + if name.endswith(".json") and not name.startswith("."): + path = os.path.join(folder, name) + try: + yield path, *_read(path) + except OSError: + continue + + +def _images(folder): + images = os.path.join(folder, IMAGES) + with contextlib.suppress(FileExistsError): + os.mkdir(images) + if not stat.S_ISDIR(os.lstat(images).st_mode): + raise OSError(f"{images} is not a folder") + return images + + +def pack(folder): + done = dict(files=0, packed=0, images=0, written=0, before=0, after=0) + with _alone(folder) as mine: + if mine is None: + return dict(done, busy=True) + images, kept = None, set() + for path, raw, st in _files(folder): + done["files"] += 1 + done["before"] += len(raw) + found = {} + + def link(m): + image = _image(m.group(1)) + if image is None: + return m.group(0) + found[image[0]] = image[1] + return b'"' + f"{IMAGES}/{image[0]}".encode() + b'"' + + packed = raw if not raw.startswith(b'{"') or raw.startswith(MARK) or LINK.search(raw) \ + else MARK + BASE64.sub(link, raw[1:]) + if not found or _expand(packed, found.__getitem__) != raw: + done["after"] += len(raw) + continue + images = images or _images(folder) + new = [name for name in found if name not in kept and _held(os.path.join(images, name), name) is None] + for name in new: + _write(os.path.join(images, name), found[name], st) + if new: + _sync(images) + kept.update(found) + done["written"] += len(new) + _write(path, packed, st) + done["packed"] += 1 + done["after"] += len(packed) + if done["packed"]: + _sync(folder) + done["images"] = len(kept) + return done + + +def restore(folder): + """A file whose image is missing or does not match its name stays packed.""" + done = dict(restored=0, failed=0) + with _alone(folder) as mine, contextlib.ExitStack() as stack: + if mine is None: + return dict(done, busy=True) + try: + images = _inner(mine, IMAGES) + stack.callback(os.close, images) + except OSError: + images = None + cache = {} + + def image(name): + if name not in cache: + cache[name] = None if images is None else _held(name, name, images) + if cache[name] is None: + raise ValueError(f"no image {name}") + return cache[name] + + for path, raw, st in _files(folder): + if not raw.startswith(MARK): + continue + try: + text = _expand(raw, image) + except ValueError: + done["failed"] += 1 + continue + _write(path, text, st) + done["restored"] += 1 + if done["restored"]: + _sync(folder) + return done + + +if __name__ == "__main__": + act = {"pack": pack, "restore": restore}.get(sys.argv[1] if len(sys.argv) > 2 else "") + if act is None: + sys.exit("usage: api_logs.py pack|restore FOLDER...") + print(json.dumps({folder: act(folder) for folder in sys.argv[2:]})) diff --git a/app/nullius/catalogue/benchmarks/osworld2/benchmark.json b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json new file mode 100644 index 0000000..c113c49 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/benchmark.json @@ -0,0 +1,82 @@ +{ + "key": "osworld2", + "title": "OSWorld2", + "category": "Computer use", + "domain": "Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a VM of its own.", + "scoring": "OSWorld-V2 evaluator", + "harness": [ + "standard" + ], + "origin": "official", + "good_to_know": [ + "Scores are partial credit: each task's own evaluate() gives 0 to 1, and the upstream's score is the mean." + ], + "limitations": [ + "Compared with the paper (Table 3): the paper ran M3 on release osworld-v2-2026.06.24, on AWS instances with all web traffic routed through a residential proxy, with 500 steps and thinking on. This entry runs release osworld-v2.1, whose task files differ from that release's for 80 of the 108 tasks, on the docker provider, with no residential proxy, and by default 100 steps and thinking off, as run_multienv_m3.sh runs it. Steps 500 and Thinking adaptive restore the paper's budget, not its release or its environment, and No hosted services is a subset of the tasks. A run counts as comparable with the paper by its model alone: read its steps and thinking beside it.", + "The upstream's M3 agent sends back every step's whole reply, its thinking included, and shortens only the history's screenshots. With Thinking adaptive through OpenRouter, pinned to MiniMax, whose window for M3 is 524,288 tokens, that history filled the window after about 220 steps. Nullius does not shorten it, which would change the agent's input. A call that overflows ends the episode with DONE and the task's evaluate() scores the VM as it is: the upstream's M3 scores the same VM after empty steps up to the step limit, while its Muse Spark stops with an error and records 0. The score's record says stopped and why.", + "Muse Spark runs as published at osworld-v2.1, with its published run's reasoning effort max and 262144 output tokens (500 steps is Steps 500). Its published score came from newer upstream code, which compacts the history near the window and asks again after an empty reply, through Meta's own API, with Muse Spark as its LLM grader: here the code has neither, the grader is Claude Sonnet 4.6 as for M3, and its scores do not compare with that row. A reply with no action goes to the simulated user as a question, or gets the default answer where the task has none, as at the tag. Its history shortens only screenshots too, in a window of 1,048,576 tokens on OpenRouter, pinned to Meta; an overflow ends the unit as for M3. Its count of input tokens before each call is skipped, since OpenRouter does not serve it.", + "The mock websites are OSWorld-web at the tag osworld-v2.1, built from source and served on the machine the units run on (websites.py in this benchmark's folder): their 40 tasks need its host suffix as Website host suffix and are refused before they run without it. Overleaf keeps one account for all: its 2 tasks run one at a time, each on Overleaf started anew, a few minutes more. GitLab is Task-Web/gitlab on the same machine, behind the sites' Caddy (gitlab_server.py in this benchmark's folder): its 2 tasks need its host name as GitLab host and its root token as the licence GITLAB_PRIVATE_TOKEN, run one at a time, and each starts GitLab anew from its snapshot, about 5 minutes more. With the VMs' writes limited, they need a GitLab write share of their own. No task of this release uses Moodle.", + "The LLM grader of 21 tasks and the simulated user of 6 are Claude Sonnet 4.6, as run_multienv_m3.sh sets them, on OpenRouter, whichever agent runs: with a custom provider those tasks fail. Muse Spark's runner raises the grader's output budget to at least 4096 tokens. The simulated user's calls count in the agent's tokens and cost. M3's prompt tells it not to ask the user, so with M3 the simulated user is almost never asked. A failed grader call leaves the unit unscored, even where it stopped the upstream's runner, unless the calls right after it repeat its request until one succeeds; it counts as infrastructure when the last call failed. A grader that refused a request carrying an image that is not whole or not PNG, JPEG, GIF or WebP (400 or 422), or over 5 MB (400, 413 or 422), keeps the upstream's score, as the upstream does, on the null control too, and the score's record says `grader_refused`: a job where the null control or two tasks carry it is read before its zeros count. Any other refusal leaves the unit unscored, as does an image the upstream cannot open at all. A refusal of the task's own reference image is taken for the agent's too. Some tasks catch another grader error themselves and score it as they choose. An evaluation that gave up reading the VM or a mock website unanswered (a timeout, a broken connection, a 502 to 504) leaves the unit unscored too, where the upstream counts it 0; its message gives the upstream's score. With Muse Spark only the task's eval.log is searched for one, not its runner's log. A Docker, VM, service or task setup that fails is a setup error, and an evaluate() that raises the score's error, in either setup; such a message names the upstream's exception by a known class only, never its words. A null control needs the provider's key too, and pays for the grader on a graded task.", + "18 tasks of No hosted services use the public internet: their outcome can change with the live sites and the machine's network.", + "Only OSWorld-V2's M3 and Muse Spark agents run, as the Original setup, through OpenRouter: M3 over its Anthropic Messages endpoint (minimax/minimax-m3), Muse Spark over its Responses endpoint (meta/muse-spark-1.3). Its contributor endpoint, whose prompts Meta may use, is not offered, the tasks being gated. The upstream's other runners are not offered. No agent of Nullius runs it yet: it needs agent.computer/v1, defined ahead of its kit.", + "The tasks and their assets are gated on Hugging Face (xlangai/osworld_v2_tasks, xlangai/osworld_v2_assets_gated, tag osworld-v2.1) and Nullius does not fetch them: put them in the upstream's evaluation_examples/task_class and cache/osworld_v2_assets before a run, with the VM image and the provider's Docker image. docs/benchmarks/osworld2-machine.md sets a machine up step by step.", + "Tasks 001, 006 and 010 ship, in the upstream's gated assets, a Thunderbird profile signed in to a live third-party mailbox, whose sign-in none of them needs and an agent here used: their units get it without its saved sign-ins, cookies and site data, its client set to start offline and fetch no mail, unlike the upstream's; its mail is kept.", + "8 of the 108 tasks ask for a proxy (3 of Offline, none of No hosted services). Each such unit gets a tinyproxy of its own on this machine, given to the upstream in place of its DataImpulse entry, whose password is a placeholder: build its image once before a run (python3 proxy.py --build, from this benchmark's folder). Their web traffic, and that of the 21 other tasks that use the public internet, leaves from this machine's address, not the paper's US residential proxy: a site that answers by country or refuses data centres can answer differently. The upstream installs tinyproxy in the VM as each of the 8 is set up, through apt, which goes out directly: the 3 of Offline then need the Ubuntu mirrors too, and each is set up more slowly.", + "Each unit runs in a VM of its own, which the upstream's docker provider gives 4 CPU cores and 4 GB of memory, and whose disk is at least 40 GB, as on the upstream's aws provider, not the docker provider's 32 GB (the VM image's whole disk, unless a task asks for more): one unit at a time is the minimum, and more run side by side (-j) as far as the machine allows. docs/benchmarks/osworld2-machine.md lists what the machine needs. The VMs run on the Docker network nullius-osworld2, 10.213.0.0/24, which Nullius makes itself: that subnet must be free on the machine. One Unix user runs a machine's VMs: the upstream's docker provider keeps its port lock in /tmp, and another user's VMs then fail to start. Task 082 takes this machine's port 3000 on 127.0.0.1 for its VM, so its units run one at a time per machine, and fail before the VM starts if another program listens on that port.", + "Tasks 103 and 104 are not supported: their evaluate() runs FreeCAD's Python on the driver's machine, which Nullius neither checks nor installs. They stay in the datasets; without FreeCAD the null control scores them 0, and an agent's unit ends in an error.", + "The VM's clock runs at UTC+8, so its day turns at 16:00 UTC: a task whose check reads the date (028 among them), set up before then and acted on after, can score 0. Start such tasks well clear of 16:00 UTC.", + "A setup step that fails is logged and the episode goes on, as the upstream's runners do by default: the score's record counts those errors of the task's last setup as `setup_errors`, and such a score can come from a VM set up only in part.", + "The upstream gives each action in the VM 90 s and does not retry a timeout: long text typed key by key, on a slow VM above all, can stop partway, and the agent sees only the next screenshot.", + "Two failures of the upstream's own code end a unit in an error with no score, and would leave the upstream's own run with no result: a screenshot the VM did not send, and a Chrome DevTools connection that times out twice in a task's evaluate(). Run such a unit again. The first is a setup error when the VM's last screenshot before it failed and the crash is the upstream reading that screenshot, of no known cause when it is another crash, otherwise the agent's error. A runner killed (signal 9, 15 or 1), as when the machine runs out of memory, or that logged a signal it did not send itself, is a setup error too. One whose episode's worker process ended without its cleanup, or that died of another signal with no traceback, is of no known cause: the machine's memory and an agent's own crash in native code leave the same trace.", + "An episode ends with DONE and is scored 10 hours after its runner starts, as at max_steps, or once a spend guard asks it to stop. Its score stays in the mean; the score's record says `stopped`, and `stopped_by` time, money or context window. A stop that comes after the episode ended by itself marks nothing; one that comes while the agent's last reply is on its way marks it even when that reply ended the episode. A runner that has not ended an hour later is stopped and the unit left unscored, as a setup error if the agent made no call; a null control a minute after its time. Either is unscored even when its runner had already written a score: run it again.", + "A unit that ends rate-limited is not run again by itself: run it again by hand.", + "A task's outcome can depend on the machine's speed: the upstream waits fixed times, not for the desktop (60 s after a task is set up, 3 s after each action), so a slow or loaded machine can show the agent a desktop that has not caught up. Compare scores from machines of similar speed and load.", + "Disk write limits apply to each VM's container on Docker's disk, not to reads or the trial's output; where Docker cannot apply them (btrfs, zfs, no io controller, cgroup v1), a unit with a limit fails before its VM starts. A limited disk slows the VM and can change a task's outcome.", + "A machine's constraints in Settings (CPUs, memory, processes) do not reach the VMs, the mock websites, GitLab or the proxies: Docker starts them outside the run's limits. Bound them with -j.", + "No reference control: OSWorld-V2 publishes no solutions. The null control is Nullius's: the upstream's run_single_example with an agent whose one action is DONE, then the task's evaluate() on the untouched VM. It runs in the score step, which this benchmark gives 3600 s per task instead of Nullius's 600 s. Task 064's setup takes longer than 600 s; with two attempts, a long setup may not fit. Its setup gets two attempts, where the upstream's runners make up to five.", + "On the untouched VM the null control scores above 0 on five tasks: 015, 076, 090 and 092 get part of their rubric's points, and 077's LLM grader gives a small score that varies between runs. Its bound, a mean of at most 0.01, only catches errors: read the null control task by task.", + "The upstream's ten lists of tasks by capability in evaluation_examples, overlapping subsets of the 108, are not datasets here.", + "Each trial keeps the upstream's output in its folder, a screenshot and an API log per step. Once the unit's processes end, the API logs keep each image once, in api_logs/images, its path in the JSON in place of its base64 (api_logs.log says how it went): python3 api_logs.py restore with the api_logs folder, from this benchmark's folder, gives the files back byte for byte. A job taken from a host before that ends can hold them partly packed: the host's folder keeps them whole, and pack or restore there finishes them." + ], + "todo": [ + "agent.computer/v1's kit, and the driver's agent side: the upstream's DesktopEnv served to any agent.", + "The upstream's other agents, beside M3 and Muse Spark." + ], + "datasets": [ + { + "key": "osworld2", + "title": "OSWorld-V2, release osworld-v2.1", + "tasks": 108, + "official": "https://github.com/xlang-ai/OSWorld-V2", + "source": "Hugging Face (gated)", + "registry": "", + "hub": "", + "verifier": "OSWorld-V2's own evaluation: each task class's evaluate() on the VM's final state, from 0 to 1, as the upstream's run_single_example writes it to result.txt.", + "access": [ + { + "step": "Ask for access to the gated datasets on Hugging Face", + "note": "xlangai/osworld_v2_tasks and xlangai/osworld_v2_assets_gated" + }, + { + "step": "Download the task classes into the upstream's evaluation_examples/task_class", + "note": "scripts/tools/download_osworld_v2_tasks.py --benchmark-release osworld-v2.1" + }, + { + "step": "Download the assets into the upstream's cache/osworld_v2_assets", + "note": "scripts/tools/download_osworld_v2_assets.py --benchmark-release osworld-v2.1" + }, + { + "step": "Fetch the VM image into the upstream's docker_vm_data", + "note": "osworld-v2-ubuntu-x86.qcow2 from xlangai/v2-image at 6e16459a, about 15 GB unpacked" + }, + { + "step": "Build the proxy image for the 8 tasks that ask for a proxy", + "note": "python3 proxy.py --build, from this benchmark's folder" + } + ], + "notes": "108 tasks; 82 in the upstream's offline list, 36 of those need no service; 63 of the 108 need no hosted service, 98 no GitLab or proxy. Code Apache-2.0; tasks and assets gated." + } + ], + "citation": "@misc{yuan26osworld2,\n title={OSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks}, \n author={Mengqi Yuan and Zilong Zhou and Xinzhuang Xiong and Weiming Wu and Jiayang Sun and Jiamin Song and Kaiqian Cui and Bowen Wang and Haoyuan Wu and Yitong Li and Dunjie Lu and Haikong Lu and Qi Zhen and Xinyuan Wang and Jiaqi Deng and Yuhao Yang and Cheng Chen and Boyuan Zheng and Alex Su and Xiao Yu and Hao Zou and Saaket Agashe and Xing Han Lu and Manpreet Kaur and Zhengyang Qi and Vincent Sunn Chen and Frederic Sala and Dayiheng Liu and Junyang Lin and Zhou Yu and Yu Su and Siva Reddy and Xin Eric Wang and Peng Qi and Tianbao Xie and Tao Yu},\n year={2026},\n eprint={2606.29537},\n archivePrefix={arXiv},\n primaryClass={cs.AI},\n url={https://arxiv.org/abs/2606.29537}, \n}", + "citation_source": "https://github.com/xlang-ai/OSWorld-V2" +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/driver.py b/app/nullius/catalogue/benchmarks/osworld2/driver.py new file mode 100644 index 0000000..4558108 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/driver.py @@ -0,0 +1,1383 @@ +"""OSWorld-V2's driver for the Nullius standard (JSON lines on stdin and stdout). + +It runs the Original setup and the null control. OSWorld-V2's own code at the +tested commit ($NULLIUS_UPSTREAM) loads, acts and scores, in processes of its +own; this driver imports none of it. Each unit's script runs in the unit's +folder, so its logs and results stay with the trial. Every process gets the +assets folder (_assets) as OSWORLD_FILE_BASE_URL and the startup folder first on +PYTHONPATH. Mock websites (websites.py), GitLab (gitlab_server.py) and a proxy +(proxy.py) are the unit's services. + +units - every task of the dataset, as domain__id +setup - the task's instruction, for the null control, from nop.py + --instruction: loading a task runs its code, so it runs apart +answer - the Original setup: the agent option's script (agents.json) on the + one task, its model calls through the unit's relay, whose stop file + ends the episode with DONE +score - the upstream's result.txt and result.json; for the null control, + nop.py's. A failed LLM grader or null-control setup, or a read of + the VM or a site that the evaluation could not make, is the + score's error, never a 0 + +Units on one machine start their VMs one after another under START_LOCK, +OSWorld1's own file. A reaper in a session of its own, armed as the script +starts, ends the script's processes once the driver lets it go or dies, +removes the unit's containers and volumes, and packs its api_logs +(api_logs.py).""" +import contextlib +import errno +import fcntl +import glob +import gzip +import importlib.util +import io +import json +import os +import queue +import re +import signal +import socket +import stat +import subprocess +import sys +import tarfile +import threading +import time +import zlib + +if sys.argv[1:2] in (["--reap"], ["--ward"]): + signal.signal(signal.SIGTERM, signal.SIG_IGN) +import gitlab_server +import proxy +import websites + +HERE = os.path.dirname(os.path.abspath(__file__)) +UP = os.environ["NULLIUS_UPSTREAM"] +_ARGS = sys.argv[1:6] if sys.argv[1:2] not in (["--reap"], ["--ward"]) else [] +_FILES = [os.path.join(HERE, n) for n in ("agents.json", "nop.py", os.path.join("startup", "sitecustomize.py"), + "websites.compose.yml", "gitlab.compose.yml")] +AGENTS_FILE, NOP, SITE, websites.OVERRIDE, gitlab_server.OVERRIDE = (_ARGS + _FILES[len(_ARGS):])[:5] +websites.SITE = SITE +with open(AGENTS_FILE, encoding="utf-8") as _f: + AGENTS = json.load(_f) +PASSWORD = AGENTS["common"][AGENTS["common"].index("--client_password") + 1] +EXAMPLES = os.path.join(UP, "evaluation_examples") +VM = os.path.join(UP, "docker_vm_data", "osworld-v2-ubuntu-x86.qcow2") +PROVIDER_IMAGE = "happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9" +ASSETS = os.path.abspath(os.path.join(UP, "cache", "osworld_v2_assets")) +RESULTS = "results" +START_LOCK = "/tmp/nullius-osworld1-vm-start.lck" +STARTED = "Started container with ports" +START_SECONDS = 1000 +SWEEPS = (2, 8, 20) +GRACE = 5 +DOCKER_SECONDS = 60 +READY_SECONDS = 120 +VOLUMES = "volumes" +VOLUME = re.compile(r"[0-9a-f]{64}") +NOTE_SECONDS = 5 +SNAPSHOT_SECONDS = 5 +NOTE_BYTES = 65536 +CLOSE_SECONDS = 60 +EPISODE_SECONDS = 36000 +EVALUATE_SECONDS = 3600 +STOPPED_BY = (("turn limit", "time"), ("context window", "context window"), ("batch cap", "money"), ("key cap", "money"), + ("forecast", "money")) +HELD_SECONDS = 300 +SETUP_FAILED = "setup_failed.json" +COLOUR = re.compile(r"\x1b\[[0-9;]*m") +RECORD = re.compile(r"^(?:\x1b\[[0-9;]*m)?\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d") +CAUSE = re.compile(r"^((?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*(?:Error|Exception|Exit|Interrupt)|ERROR)(?::|$)") +FRAME = re.compile(r'^ File ".*[/\\]desktop_env[/\\]desktop_env\.py", line \d+, in (\w+)$') +INNER = re.compile(r'^ File "(?:.*[/\\])?([^/\\"]*)", line \d+, in (\S+)$') +LISTS = { + "offline36": "001 002 004 010 012 022 029 030 040 042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 " + "076 080 084 085 088 094 103 104 105 106 107 108".split(), + "no_services63": "001 002 003 004 006 009 010 011 012 014 017 022 023 024 025 027 029 030 032 033 034 040 " + "042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 071 074 076 078 080 083 084 " + "085 086 087 088 089 090 092 093 094 095 096 097 099 103 104 105 106 107 108".split(), + "websites35": "005 007 008 013 015 016 018 019 020 021 028 031 035 038 039 043 046 052 057 058 060 065 068 069 " + "070 072 073 077 079 081 082 091 100 101 102".split(), +} +LISTS["websites98"] = sorted(LISTS["no_services63"] + LISTS["websites35"]) +JUDGED = frozenset("003 007 008 019 025 031 035 041 070 074 075 077 078 079 081 082 087 089 090 092 093".split()) +ASKS_USER = frozenset("007 024 026 034 095 098".split()) +RETRIES_SITE_READS = frozenset({"069"}) +GUEST_PORTS = {"082": (3000,)} +MAILBOXES = {"001": ("task_001_thunderbird-profile_fixed.tar.gz",), "006": ("thunderbird-profile.tar.gz",), + "010": ("email_010.tar.gz",)} +SIGN_INS = re.compile(r"(?:^|/)(?:logins[^/]*\.json|signons[^/]*|cookies\.sqlite[^/]*|user\.js)$|/storage/default(?:/|$)") +ONLINE = re.compile(r'^(\s*user_pref\("mail\.server\.\w+\.(?:login_at_startup|check_new_mail|download_on_biff)",\s*)' + r'true(\s*\);)[ \t\r]*$', re.M) +OFFLINE = 'user_pref("offline.autoDetect", false);\nuser_pref("offline.startup_state", 3);\n' +PORT_LOCK = "/tmp/nullius-osworld2-port-{}.lck" +USER_ROUTE = "OSWORLD_USER_SIM_BASE_URL" +GRADER_FAILED = ("Error in compare_images_with_llm: ", "Error in compare_image_edit_with_llm: ", + "Error in _compare_answers_with_llm: ", "Error in compare_text_with_llm: ") +JUDGE_ROUTE = re.compile(r"/([A-Za-z0-9_-]{1,64}-judge)/v1/?$") +IMAGE_TYPES = ("image/png", "image/jpeg", "image/gif", "image/webp") +IMAGE_BYTES = 5 * 1024 * 1024 +GRADER_STATUS = re.compile(r"\b(?:Error code|[Ss]tatus(?: code)?|HTTP)\s*[:=]?\s*([1-5][0-9]{2})\b") +GRADER_NAME = re.compile(r"^((?:[A-Za-z_]\w*\.)*)([A-Za-z_]\w*)(?=:|\s|$)") +_SDK = ("APIError APIStatusError APIConnectionError APITimeoutError APIResponseValidationError BadRequestError " + "AuthenticationError PermissionDeniedError NotFoundError ConflictError UnprocessableEntityError RateLimitError " + "InternalServerError OverloadedError ServiceUnavailableError DeadlineExceededError " + "LengthFinishReasonError ContentFilterFinishReasonError").split() +_HTTPX = ("HTTPError RequestError TransportError TimeoutException ConnectTimeout ReadTimeout WriteTimeout PoolTimeout " + "NetworkError ConnectError ReadError WriteError CloseError ProtocolError LocalProtocolError RemoteProtocolError " + "ProxyError UnsupportedProtocol DecodingError TooManyRedirects HTTPStatusError InvalidURL").split() +_REQUESTS = ("RequestException HTTPError ConnectionError ProxyError SSLError Timeout ConnectTimeout ReadTimeout " + "URLRequired TooManyRedirects MissingSchema InvalidSchema InvalidURL InvalidHeader ChunkedEncodingError " + "ContentDecodingError StreamConsumedError RetryError JSONDecodeError").split() +_BUILTIN = sorted(n for n, v in vars(__import__("builtins")).items() if isinstance(v, type) and issubclass(v, BaseException)) +GRADER_CLASSES = {"anthropic": _SDK, "openai": _SDK, "httpx": _HTTPX, "requests": _REQUESTS, + "requests.exceptions": _REQUESTS, "builtins": _BUILTIN, + "": sorted(set(_SDK + _HTTPX + _REQUESTS + _BUILTIN))} +GRADER_RECORD = re.compile(r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? ERROR " + r"(?:llm_metrics/\d+-[^\]]*|desktopenv\.env)\] (Error in \w+: )(.*)$") +HEAD = r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? \w+ " +SET_UP = re.compile(HEAD + r"[^\]]*\] Environment (?:has been used, reverting to snapshot|is clean, skipping snapshot revert)") +SET_UP_DONE = re.compile(HEAD + r"[^\]]*\] Environment setup complete\.$") +SETUP_ERROR = re.compile(r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(?:,\d+)? ERROR (?:desktopenv\.)?setup(?:/\d+-[^\]]*)?\] (.*)$") +SETUP_RETRY = re.compile(r"retrying|Upload attempt \d+ failed|Failed to upload file \S", re.I) +STEPPED = re.compile(HEAD + r"(?:lib_run_single|desktopenv\.experiment)(?:/\d+-[^\]]*)?\] " + r"(?:Done:|(?:Phase \d+ )?(?:Step (\d+)|User simulator Q):)") +SIGNALLED = re.compile(HEAD + r"[^\]]*-MainProcess\] Received signal (\d+)\. Gracefully shutting down\.\.\.$") +MAIN_ERROR = re.compile(HEAD + r"[^\]]*-MainProcess\] Unexpected error in main process: ") +WORKER = re.compile(HEAD + r"[^\]]*\] (?:Process (\S+) started\.|(\S+) cleaning up environment\.\.\.)$") +SCREENSHOT = re.compile(HEAD + r"(?:desktopenv\.pycontroller|python)(?:/\d+-[^\]]*)?\] " + r"(Got screenshot successfully|Failed to get screenshot\.)$") +RUNNER = re.compile(HEAD + r"(?:lib_run_single/\d+-[^\]]*|desktopenv\.experiment)\] (?:(?PDone:|(?:Phase (?P\d+) )?(?:Step \d+|User simulator Q):)" + r"|(?PStarting phase \d+ )|(?:Multi-phase r|R)esult: -?\d)") +CONTROLLER = re.compile(HEAD + r"(?:python/\d+-[^\]]*|desktopenv\.pycontroller)\] (.*)$") +THINGS = "file|screenshot|accessibility tree|terminal output|screen size|window size|wallpaper|desktop path|directory tree" +READ = re.compile(rf"An error occurred while trying to get (?:the )?(?P{THINGS}): |Failed to get (?P{THINGS})\. " + rf"Status code: (?P\d+)$|Retrying to get (?:the )?(?P{THINGS})\.$|Failed to get (?P{THINGS})\.$") +STATE = re.compile(HEAD + r"(?:state/\d+-[^\]]*|desktopenv\.getters\.state)\] \[GET_STATE\w*\] (?:Failed to " + r"(fetch state|download file) from \S+: (.*)$)") +SITE_STATUS = re.compile(r"\b(\d{3}) (?:Client|Server) Error\b") +GATEWAY = ("502", "503", "504") +SETUP_DOC = "docs/benchmarks/osworld2-machine.md" +WRITE_OPTIONS = ("NULLIUS_OPTION_WRITE_MBPS", "NULLIUS_OPTION_WRITE_IOPS") +PROBE = "nullius-osworld2-probe-" +PROBE_SECONDS = 600 +PACK_LOG = "api_logs.log" +_STARTUP = [] +_PORTS = [] +_ARMED = {} +_OWN = [] + + +def _unit(unit): + domain, _, eid = unit.partition("__") + return domain, eid + + +def units(dataset=None, setup=None): + if not os.environ.get("NULLIUS_DATASET"): + name = os.environ.get("NULLIUS_DATASET_ID") or dataset or "" + if name not in LISTS: + raise RuntimeError(f"no tasks file for the dataset {name or '(none)'}, and no list of its own") + return {"units": [f"tasks__{e}" for e in LISTS[name]]} + with open(os.environ["NULLIUS_DATASET"], encoding="utf-8") as f: + meta = json.load(f) + return {"units": [f"{d}__{e}" for d, ids in meta.items() for e in ids]} + + +def _task_file(domain, eid): + path = os.path.join(EXAMPLES, "task_class", f"task_{eid}.py") + if not os.path.isfile(path): + raise RuntimeError(f"no task class for {domain}__{eid} at {path}: put the gated task classes in " + f"{os.path.dirname(path)} before a run ({SETUP_DOC})") + return path + + +def _task(dir, domain, eid): + out = os.path.join(dir, "instruction.json") + _run([sys.executable, NOP, "--instruction", EXAMPLES, domain, eid, out], + dir, _env(eid), os.path.join(dir, "setup.log"), "the task's loading") + with open(out, encoding="utf-8") as f: + task = json.load(f) + os.remove(out) + return task + + +def _letting_go(handler): + def let_go(*a, **kw): + try: + return handler(*a, **kw) + except BaseException: + while _OWN: + os.close(_OWN.pop()) + raise + return let_go + + +@_letting_go +def setup(unit, dir, **_): + domain, eid = _unit(unit) + _fresh(dir) + _task_file(domain, eid) + websites.needed(eid) + gitlab_server.needed(eid) + _workspace(dir) + return {"prompt": _task(dir, domain, eid)["instruction"]} + + +LINES = queue.Queue() +CLOSED = threading.Event() + + +class _Waiting: + """What a wait before the runner ends on: the unit's pipe closed, or its turn limit passed.""" + def __init__(self, until=None): + self.until = until + + def wait(self, seconds): + return CLOSED.wait(seconds) or bool(self.until and time.time() > self.until) + + +def _read(): + for line in sys.stdin: + LINES.put(line) + CLOSED.set() + LINES.put(None) + + +def _arm(pgid, dir, vms): + """A reaper that starts once this process closes its pipe or dies; the locks it is given stay held until it ends.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--reap", str(pgid), dir, "vms" if vms else "group"], + stdin=r, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, + pass_fds=websites.held_fds() + tuple(_PORTS) + tuple(_OWN)) + except BaseException: + os.close(w) + raise + finally: + os.close(r) + _ARMED[pgid] = w + + +def _reaper(pgid, dir, vms, seen=()): + """``seen``: the volumes this driver saw.""" + if pgid not in _ARMED: + _arm(pgid, dir, vms) + with contextlib.suppress(OSError), os.fdopen(_ARMED.pop(pgid), "w") as f: + f.write(" ".join(v for v in dict.fromkeys(seen) if VOLUME.fullmatch(v))) + + +def _stop(p, log, what, vms, seen=()): + """The reaper takes the process's group over. + The upstream's own signal handlers are not relied on: they skip the provider's cleanup while a VM boots.""" + _reaper(p.pid, os.path.dirname(log), vms, seen) + raise RuntimeError(f"{what} was stopped: the unit's time ran out") + + +def _alive(pgid): + """A process that waits for its parent to see its exit does not count, so a stopped script ends its group.""" + try: + os.killpg(pgid, 0) + except OSError: + return False + try: + pids = [p for p in os.listdir("/proc") if p.isdigit()] + except OSError: + return True + for pid in pids: + try: + with open(f"/proc/{pid}/stat", encoding="utf-8", errors="replace") as f: + state, _, group = f.read().rpartition(")")[2].split()[:3] + except (OSError, ValueError): + continue + if group == str(pgid) and state not in ("Z", "X"): + return True + return False + + +def _gone(pgid, seconds): + t0 = time.time() + while time.time() - t0 < seconds: + if not _alive(pgid): + return True + time.sleep(0.2) + return False + + +def reap(pgid, dir, vms, seen=()): + """SIGTERM to the group, GRACE seconds, SIGKILL; then the unit's containers removed and its api_logs packed. + Volumes are read before the signals: the upstream removes a container without its volume.""" + seen = [v for v in seen if VOLUME.fullmatch(v)] + if vms: + done = threading.Event() + done.set() + snapshot = threading.Thread(target=_noting, args=(dir, done, seen), daemon=True) + snapshot.start() + snapshot.join(SNAPSHOT_SECONDS) + ended = False + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(pgid, sig) + except ProcessLookupError: + ended = True + break + except OSError: + break + if _gone(pgid, GRACE): + ended = True + break + packing = threading.Thread(target=_pack, args=(dir,)) + if ended: + packing.start() + if vms: + for wait in SWEEPS: + time.sleep(wait) + try: + _remove_vms(dir, seen) + except (RuntimeError, OSError): + pass + try: + _old_probes() + except (RuntimeError, OSError): + pass + if ended: + packing.join() + + +def _pack(dir): + """A failure leaves the folder as it is and the unit's result alone; each outcome is a line in PACK_LOG.""" + try: + folders = glob.glob(os.path.join(glob.escape(dir), RESULTS, "**", "api_logs"), recursive=True) + if folders: + import api_logs + except Exception as e: + _packed(dir, f"api_logs not packed: {type(e).__name__}: {e}") + return + for folder in folders: + try: + done = api_logs.pack(folder) + except Exception as e: + done = {"error": f"{type(e).__name__}: {e}"} + _packed(dir, f"{os.path.relpath(folder, dir)} {json.dumps(done)}") + + +def _packed(dir, line): + try: + fd = os.open(os.path.join(dir, PACK_LOG), os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o644) + with os.fdopen(fd, "a", encoding="utf-8") as f: + f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + line + "\n") + except OSError: + pass + + +class EnvironmentSetupError(RuntimeError): + """The null control's setup failed for good: the score's error, in the driver's own words only.""" + + +class CancelledError(RuntimeError): + pass + + +class CauseUnknownError(RuntimeError): + """Neither the agent's nor the environment's, as far as the log shows.""" + + +class VerifierError(RuntimeError): + """The task's evaluate() raised: the score's error.""" + + +class NoAnswer(RuntimeError): + """A docker command its daemon did not answer in time.""" + + +def _docker(*args, seconds=DOCKER_SECONDS): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds) + except subprocess.TimeoutExpired: + raise NoAnswer(f"docker {args[0]} got no answer in {seconds} s: is its daemon running?") + except OSError: + raise RuntimeError("no docker on this machine's PATH: the upstream's docker provider needs it") + + +def _asked(*args): + """Waits READY_SECONDS, since a failed check before a VM start costs the unit.""" + try: + return _docker(*args, seconds=READY_SECONDS) + except NoAnswer as e: + command = " ".join(a for a in args[:2] if not a.startswith("-")) + raise NoAnswer(f"Docker's daemon did not answer docker {command} in {READY_SECONDS} s: it may be busy; " + "try again or run fewer units at once") from e + + +def _mine(dir): + """The containers that mount the unit's image link, found by the provider image's digest, and their anonymous volumes.""" + mine = os.path.join(os.path.abspath(dir), "vm") + os.sep + every = _docker("ps", "-a", "-q", "--no-trunc", "--filter", f"ancestor={PROVIDER_IMAGE}").stdout.split() + if not every: + return [], [] + out = _docker("inspect", "-f", "{{.Id}}\t{{json .Mounts}}", *every).stdout + ids, volumes = [], [] + for cid, _, mounts in (ln.partition("\t") for ln in out.splitlines() if ln.strip()): + try: + mounts = [m for m in json.loads(mounts) or [] if isinstance(m, dict)] + except ValueError: + continue + if any(str(m.get("Source", "")).startswith(mine) for m in mounts): + ids.append(cid) + volumes += [m["Name"] for m in mounts if m.get("Type") == "volume" and VOLUME.fullmatch(str(m.get("Name")))] + return ids, volumes + + +def _noted(dir, volumes=()): + """Notes each anonymous volume while its container runs: the upstream removes containers without their volumes. + vm/volumes is read and written only as a plain file of this user's with one link.""" + volumes = [v for v in dict.fromkeys(volumes) if VOLUME.fullmatch(v)] + try: + fd = os.open(os.path.join(dir, "vm", VOLUMES), + os.O_RDWR | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + except OSError: + return volumes + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or st.st_nlink != 1: + return volumes + fcntl.flock(fd, fcntl.LOCK_EX) + known = list(dict.fromkeys(v for v in os.pread(fd, NOTE_BYTES, 0).decode("utf-8", "replace").split() + if VOLUME.fullmatch(v))) + new = [v for v in volumes if v not in known] + if new: + os.write(fd, "".join(v + "\n" for v in new).encode()) + return known + new + except OSError: + return volumes + finally: + os.close(fd) + + +def _noting(dir, done, seen): + """Beside the wait for the process, so a slow docker never holds the unit's stop.""" + while True: + try: + found = _mine(dir)[1] + seen.extend(found) + _noted(dir, found) + except (RuntimeError, OSError): + pass + if done.wait(NOTE_SECONDS): + return + + +def _remove_vms(dir, seen=None): + seen = [] if seen is None else seen + ids, volumes = _mine(dir) + seen.extend(volumes) + noted = _noted(dir, volumes) + if ids: + _docker("rm", "-f", "-v", *ids) + gone = [v for v in dict.fromkeys(noted + seen) if VOLUME.fullmatch(v)] + if gone: + _docker("volume", "rm", *gone) + proxy.remove(dir) + + +def _started(p, log, what, until=None): + """Until the VM container runs, the process ends, START_SECONDS pass or ``until``, so the next start does not queue.""" + t0 = time.time() + while p.poll() is None and time.time() - t0 < START_SECONDS and not (until and time.time() > until): + if CLOSED.wait(1): + _stop(p, log, what, True) + try: + with open(log, encoding="utf-8", errors="replace") as f: + if STARTED in f.read(): + return + except OSError: + pass + + +def _setup_failed(failed): + """Types and attempts only, never the message, which could quote the task.""" + try: + with open(failed, encoding="utf-8") as f: + said = json.load(f) + except FileNotFoundError: + return None + except (OSError, ValueError): + said = {} + said = said if isinstance(said, dict) else {} + cause = said.get("cause") + return (f"the environment's setup failed ({said.get('error') or 'no type'}" + + (f" from {cause}" if cause else "") + f", after {said.get('attempts') or '?'} attempt" + + ("" if said.get("attempts") == 1 else "s") + "), and the task was not scored") + + +def _run(argv, cwd, env, log, what, stagger=False, failed=None, until=None, grace=EVALUATE_SECONDS): + """``argv`` in a session of its own, handed to the reaper when it ends or the unit is stopped. + Once ``failed`` exists, the run is an EnvironmentSetupError in the driver's words: the log can quote the task.""" + lock = None + if stagger: + with _set_up(): + lock = os.fdopen(websites.opened(START_LOCK)) + while True: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if _Waiting(until).wait(1): + lock.close() + raise EnvironmentSetupError(f"{what} was not started: the unit's time ran out") + try: + with open(log, "a") as f: + p = subprocess.Popen(argv, cwd=cwd, env=env, stdout=f, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + start_new_session=True) + until = until and min(until, time.time() + EPISODE_SECONDS) + try: + _arm(p.pid, cwd, stagger) + armed = None + except Exception as e: + armed = e + if lock and not armed: + _started(p, log, what, until) + except OSError as e: + raise EnvironmentSetupError(f"{what} did not start: {type(e).__name__}: {e}") from None + finally: + if lock: + lock.close() + if armed: + try: + reap(p.pid, cwd, stagger) + except Exception: + with contextlib.suppress(OSError): + os.killpg(p.pid, signal.SIGKILL) + if stagger: + with contextlib.suppress(RuntimeError, OSError): + _remove_vms(cwd) + raise EnvironmentSetupError(f"{what}'s reaper did not start: {type(armed).__name__}: {armed}") + done, seen, given, hard = threading.Event(), [], None, None + try: + if stagger: + threading.Thread(target=_noting, args=(cwd, done, seen), daemon=True).start() + while p.poll() is None: + if CLOSED.wait(1): + _stop(p, log, what, stagger, seen) + if until and time.time() > until: + hard = hard or time.time() + grace + try: + os.makedirs(os.path.dirname(_stop_file(cwd)), exist_ok=True) + with open(_stop_file(cwd), "a") as f: + f.write("turn limit\n") + until = None + except OSError: + pass + if hard and time.time() > hard: + _reaper(p.pid, cwd, stagger, seen) + raise (VerifierError if _acted(cwd) or failed and _set_up_done(log) else EnvironmentSetupError)( + f"{what} did not end {grace} s after the unit's turn limit, and was handed to the reaper") + if failed and given is None and os.path.exists(failed): + given = time.time() + if given is not None and time.time() - given > CLOSE_SECONDS: + _reaper(p.pid, cwd, stagger, seen) + raise EnvironmentSetupError(f"{what} did not end {CLOSE_SECONDS} s after its failure, and was " + f"handed to the reaper: {_setup_failed(failed)} (the whole log: {log})") + finally: + done.set() + if p.pid in _ARMED: + _reaper(p.pid, cwd, stagger, seen) + if stagger: + try: + _remove_vms(cwd, seen) + except (RuntimeError, OSError): + pass + said = _setup_failed(failed) if failed else None + if said: + raise EnvironmentSetupError(f"{what} exited {p.returncode}: {said} (the whole log: {log})") + if p.returncode != 0: + kind, said = _tail(log, p.returncode) + raise kind(f"{what} exited {p.returncode}: {said}") + + +def _tail(log, code=0): + """The error class the log's last traceback makes, and that traceback's exception by a known class's name only. + A runner killed, or ended by a signal it did not send itself, is the environment's; one whose worker process never + reached its cleanup, or that died of another signal with no traceback, is of no known cause.""" + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip() for ln in f.read().splitlines()] + except OSError: + return RuntimeError, f"no log at {log}" + start = max((i for i, ln in enumerate(lines) if ln.endswith("Traceback (most recent call last):")), default=len(lines)) + if any(map(RUNNER.match, lines[start:])): + start = len(lines) + signalled = [m.group(1) for m in map(SIGNALLED.match, lines) if m] + if code in (-signal.SIGKILL, -signal.SIGTERM, -signal.SIGHUP) or code >= 0 and signalled and not any(map(MAIN_ERROR.match, lines)): + return EnvironmentSetupError, f"ended by signal {-code if code < 0 else signalled[-1]} (the whole log: {log})" + workers = [m.groups() for m in map(WORKER.match, lines) if m] + if code == 0 and {a for a, _ in workers if a} - {b for _, b in workers} or code < 0 and start == len(lines): + return CauseUnknownError, (f"{f'ended by signal {-code}, with no traceback' if code else 'its worker process ended without its cleanup'}" + f" (the whole log: {log})") + block = lines[start:] + raised = next((ln for ln in block[1:] if ln and not ln[0].isspace()), "") + named = GRADER_NAME.match(raised) + module, name = (named.group(1).rstrip("."), named.group(2)) if named else ("", "") + frames = {m.group(1) for m in map(FRAME.match, block) if m} + said = name if name == "EnvironmentSetupError" or name in GRADER_CLASSES.get(module.replace("._exceptions", ""), ()) \ + else "an exception of the upstream's own" if name else "no exception named" + kind = EnvironmentSetupError if name == "EnvironmentSetupError" or frames & {"reset", "_start_emulator", "__init__"} \ + else VerifierError if frames & {"evaluate", "_call_task_evaluate"} else RuntimeError + shots = [m.group(1) for m in map(SCREENSHOT.match, lines[:start]) if m] + if kind is RuntimeError and name and shots[-1:] == ["Failed to get screenshot."]: + inner = ([m.groups() for m in map(INNER.match, block) if m] or [("", "")])[-1] + step = [m.group(1) for m in map(STEPPED.match, lines[:start]) if m and m.group(1)][-1:] + where = f"in step {step[0]}" if step else "before the first step" + if name == "UnidentifiedImageError" or inner[1] == "_screenshot_bytes" \ + or name == "TypeError" and "NoneType" in raised and inner[0] == "lib_run_single.py": + return EnvironmentSetupError, f"the VM stopped answering {where}: its last screenshot failed (the whole log: {log})" + return CauseUnknownError, f"{said}, after the VM's last screenshot failed {where} (the whole log: {log})" + return kind, f"{said} (the whole log: {log})" + + +def _env(eid, assets=ASSETS, **extra): + """GitLab's token only for a task that needs it.""" + kept = {k: v for k, v in os.environ.items() if k != gitlab_server.TOKEN or eid in gitlab_server.TASKS} + return dict(kept, PYTHONPATH=os.pathsep.join([os.path.dirname(os.path.abspath(SITE)), UP]), + PYTHONIOENCODING="utf-8", OSWORLD_FILE_BASE_URL=assets, **websites.env(), **gitlab_server.env(eid), + **extra) + + +def _assets(dir, eid): + """The assets, with the task's mail profiles signed out of their live mailboxes and started offline.""" + if eid not in MAILBOXES: + return ASSETS + own, root = f"task_{eid}", os.path.join(dir, "assets") + names = os.listdir(os.path.join(ASSETS, own)) + missing = sorted(set(MAILBOXES[eid]) - set(names)) + if missing: + raise RuntimeError(f"no {', '.join(missing)} in {os.path.join(ASSETS, own)}") + os.makedirs(os.path.join(root, own), exist_ok=True) + for src, dst in [(os.path.join(ASSETS, n), os.path.join(root, n)) for n in os.listdir(ASSETS) if n != own] + \ + [(os.path.join(ASSETS, own, n), os.path.join(root, own, n)) for n in names if n not in MAILBOXES[eid]]: + if not os.path.lexists(dst): + os.symlink(src, dst) + for name in MAILBOXES[eid]: + _signed_out(os.path.join(ASSETS, own, name), os.path.join(root, own, name)) + return root + + +def _signed_out(src, dst): + with tarfile.open(src) as old, open(dst + ".part", "wb") as raw, \ + gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as gz, tarfile.open(fileobj=gz, mode="w") as new: + for m in old: + name = os.path.normpath(m.name) + if m.islnk() or m.issparse() or os.path.isabs(name) or name.split(os.sep)[0] == "..": + raise RuntimeError(f"{os.path.basename(src)} has {m.name}, which this driver does not rewrite") + if SIGN_INS.search(name): + continue + data = old.extractfile(m) if m.isfile() else None + if data and name.endswith("/prefs.js"): + text = (ONLINE.sub(r"\1false\2", data.read().decode("utf-8")).rstrip("\n") + "\n" + OFFLINE).encode("utf-8") + data, m.size = io.BytesIO(text), len(text) + m.pax_headers.pop("size", None) + new.addfile(m, data) + os.replace(dst + ".part", dst) + + +def _port_free(eid, port): + """Docker's proxy and this bind both set SO_REUSEADDR, so a connection closing there is no refusal. + localhost can be ::1 first, so a listener there would answer the task.""" + for family, host in ((socket.AF_INET, "127.0.0.1"), (socket.AF_INET6, "::1")): + try: + with socket.socket(family, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + except OSError as e: + if e.errno == errno.EADDRINUSE: + raise RuntimeError(f"task {eid} reaches its VM's port {port} at this machine's localhost:{port}, " + f"and another process listens on {host} there: free that port, or run the " + "task on another machine") from None + + +@contextlib.contextmanager +def _published(eid, closed=_Waiting()): + """The task's GUEST_PORTS, for one VM at a time on this machine; the reaper keeps the locks until that VM is gone.""" + fds = [] + try: + for port in GUEST_PORTS.get(eid, ()): + try: + fds.append(websites.opened(PORT_LOCK.format(port))) + except OSError as e: + raise RuntimeError(f"the lock {PORT_LOCK.format(port)} cannot be opened ({type(e).__name__}): " + "remove it if it is not a plain file") from None + while True: + try: + fcntl.flock(fds[-1], fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if closed.wait(1): + raise RuntimeError(f"the unit's time ran out while another unit's VM held port {port}") + _port_free(eid, port) + _PORTS.extend(fds) + yield {_startup().PORTS: " ".join(map(str, GUEST_PORTS[eid]))} if fds else {} + finally: + for fd in fds: + if fd in _PORTS: + _PORTS.remove(fd) + os.close(fd) + + +def _base(url): + return url[:-3] if url.endswith("/v1") else url + + +def _graded(eid, judge): + """The LLM grader's settings from the run's judge route; a judged task with no route is refused before its VM starts.""" + if not judge: + if eid in JUDGED: + raise RuntimeError(f"task {eid} is scored by the LLM grader and this run gives it no judge route: " + "run it with the benchmark's judge (score.judge)") + return {} + if not isinstance(judge, dict) or not all(isinstance(judge.get(k), str) and judge[k] for k in ("model", "base_url", "key")): + raise RuntimeError("the judge route needs a model, a base_url and a key") + return {"OSWORLD_EVAL_MODEL_PROVIDER": "anthropic", "OSWORLD_EVAL_MODEL_NAME": judge["model"], + "OSWORLD_EVAL_MODEL_BASE_URL": _base(judge["base_url"]), "OSWORLD_EVAL_MODEL_API_KEY": judge["key"]} + + +def _asks_user(eid, name, spec): + """A task with a simulated user and no route for it is refused before its VM starts.""" + if eid in ASKS_USER and not spec.get("env", {}).get(USER_ROUTE): + raise RuntimeError(f"task {eid} has a simulated user and the {name} agent gives it no route: " + f"set {USER_ROUTE} in its agents.json entry") + + +def vm_image(): + """A unit never fetches the image: an interrupted unpacking leaves a broken one.""" + if not os.path.isfile(VM): + raise RuntimeError(f"no VM image at {VM}: put osworld-v2-ubuntu-x86.qcow2 there before a run ({SETUP_DOC})") + return VM + + +def _nat(): + try: + with open("/proc/modules", encoding="utf-8") as f: + if any(ln.split(" ", 1)[0] == "iptable_nat" for ln in f): + return True + except OSError: + pass + return os.path.isdir("/sys/module/iptable_nat") + + +def _provider_image(): + """Pulled inside the provider's port lock, it would hold the next units' starts past their wait.""" + if _asked("image", "inspect", PROVIDER_IMAGE).returncode != 0: + raise RuntimeError(f"no {PROVIDER_IMAGE} on this machine: docker pull {PROVIDER_IMAGE} before a run ({SETUP_DOC})") + + +def _startup(): + """Loaded under another name, so it wraps nothing in this process.""" + if not _STARTUP: + spec = importlib.util.spec_from_file_location("osworld2_startup", SITE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _STARTUP.append(module) + return _STARTUP[0] + + +def _last(said, otherwise): + return ((said or "").strip().splitlines() or [otherwise])[-1] + + +def _old_probes(ready=False): + """This user's probes whose driver is gone or older than PROBE_SECONDS; another user's stay.""" + docker = _asked if ready else _docker + names = docker("ps", "-a", "--filter", f"name={PROBE}{os.getuid()}-", "--format", "{{.Names}}").stdout.split() + old = [] + for name in names: + m = re.fullmatch(re.escape(f"{PROBE}{os.getuid()}-") + r"(\d+)-(\d+)", name) + if not m: + continue + try: + os.kill(int(m.group(1)), 0) + alive = True + except ProcessLookupError: + alive = False + except OSError: + alive = True + if not alive or time.time_ns() - int(m.group(2)) > PROBE_SECONDS * 10 ** 9: + old.append(name) + if old: + docker("rm", "-f", *old) + + +def _probe_name(): + return f"{PROBE}{os.getuid()}-{os.getpid()}-{time.time_ns()}" + + +def _write_limit(): + """Proof before a VM start, by a probe container, that a set write limit holds: a daemon that cannot limit drops it. + An unanswered probe create is tried once more under a new name.""" + if os.environ.get("NULLIUS_OPTION_VM_PROVIDER", "docker") != "docker" \ + or all(os.environ.get(n, "none") == "none" for n in WRITE_OPTIONS): + return + if not os.path.isfile(SITE): + raise RuntimeError(f"the VM's disk write limit cannot be applied: no {SITE}, which sets it in the unit's " + "processes; set Disk write MB/s and Disk write IOPS to none, or restore the file") + site = _startup() + limits = site.wanted(os.environ) + cannot = "the VM's disk write limit cannot be applied: " + if importlib.util.find_spec("docker") is None: + raise site.WriteLimitError(cannot + f"no docker-py in {sys.executable}'s environment, which the upstream's " + "docker provider needs too: install the driver's environment again") + info = _asked("info", "--format", "{{.DockerRootDir}}\t{{.CgroupVersion}}") + root, _, cgroup = (part.strip() for part in info.stdout.partition("\t")) + if info.returncode != 0 or not root: + raise site.WriteLimitError(cannot + "docker info gave no root (" + _last(info.stderr, "no answer") + + "): is Docker's daemon running and usable by this user?") + if cgroup != "2": + raise site.WriteLimitError(cannot + f"Docker runs on cgroup v{cgroup or '?'}, where buffered writes pass a " + f"device limit; {site.ACTION}") + device = site.root_disk(root) + _old_probes(ready=True) + flags = [x for _, rate, _, flag in limits for x in (flag, f"{device}:{rate}")] + + def create(name): + return _asked("create", "--name", name, "--network", "none", "--mount", "type=tmpfs,destination=/storage", + *flags, PROVIDER_IMAGE) + + probe = _probe_name() + try: + try: + made = create(probe) + except NoAnswer as e: + if _asked("rm", "-f", probe).returncode != 0: + raise e + probe = _probe_name() + made = create(probe) + if made.returncode != 0: + raise site.WriteLimitError(cannot + "a probe container was refused: " + _last(made.stderr, "no reason") + + f"; {site.ACTION}") + try: + host = json.loads(_asked("inspect", "-f", "{{json .HostConfig}}", probe).stdout or "{}") or {} + except ValueError: + host = {} + finally: + try: + _asked("rm", "-f", probe) + except RuntimeError: + pass + dropped = [field for _, rate, field, _ in limits if {"Path": device, "Rate": rate} not in (host.get(field) or [])] + if dropped: + said = _last(made.stderr, "") + raise site.WriteLimitError(cannot + f"Docker kept no {', '.join(dropped)} on {device}" + + (f" ({said})" if said else "") + f"; {site.ACTION}") + + +def _ready(domain, eid): + """What a VM start of the task needs, checked before one, so it does not fail minutes later; no /dev/kvm is a warning.""" + image = vm_image() + _task_file(domain, eid) + sites = websites.needed(eid) + lab = gitlab_server.needed(eid) + _startup().network(_asked) + if sites: + websites.ready(sites, state=False) + if lab: + gitlab_server.prepared() + if not os.path.isdir(ASSETS) or not os.listdir(ASSETS): + raise RuntimeError(f"no assets at {ASSETS}: put the gated assets there before a run ({SETUP_DOC})") + _provider_image() + _write_limit() + if not _nat(): + raise RuntimeError("the kernel module iptable_nat is not loaded, and the VM's port forwarding needs it: " + "sudo modprobe ip_tables iptable_nat") + if not os.path.exists("/dev/kvm"): + print("no /dev/kvm: the VM runs without KVM, slowly, and a slow VM can change a task's outcome", + file=sys.stderr, flush=True) + return image + + +@contextlib.contextmanager +def _gitlab(eid, dir, closed=_Waiting()): + """GitLab started from its snapshot for a task of gitlab_server.TASKS, its warden first, however the unit ends.""" + value = gitlab_server.needed(eid) + with gitlab_server.held(value, closed): + if not value: + yield + return + where = gitlab_server.prepared() + limits = gitlab_server.unit(where, dir) + with _warden(dir): + gitlab_server.up(where, limits, closed) + yield + + +@contextlib.contextmanager +def _warden(dir): + """The warden removes GitLab once this process closes its pipe or dies, then lets the locks go.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--ward", str(r), dir], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, + pass_fds=websites.held_fds() + (r,)) + finally: + os.close(r) + try: + yield + finally: + os.close(w) + + +def _disk(): + """The disk the VMs' writes are limited on, for the proxy's own limit; None with no limit.""" + if all(os.environ.get(n, "none") == "none" for n in WRITE_OPTIONS): + return None + site = _startup() + site.wanted(dict(os.environ, **proxy.TAKEN)) + return site.root_disk(_asked("info", "--format", "{{.DockerRootDir}}").stdout.strip()) + + +def _workspace(dir): + """The unit's folder as the upstream's root for the paths its tasks name.""" + link = os.path.join(dir, "evaluation_examples") + if not os.path.lexists(link): + os.symlink(EXAMPLES, link) + os.makedirs(os.path.join(dir, "logs"), exist_ok=True) + + +def _vm(dir, domain, eid): + """The unit's link to the VM image, which the provider mounts by its path: the reaper's mark.""" + image = _ready(domain, eid) + _workspace(dir) + vm = os.path.join(dir, "vm", os.path.basename(VM)) + os.makedirs(os.path.dirname(vm), exist_ok=True) + if not os.path.lexists(vm): + os.symlink(image, vm) + return vm + + +def _agent(): + name = os.environ.get("NULLIUS_OPTION_AGENT", "m3") + if name not in AGENTS["agents"]: + raise RuntimeError(f"no agent {name} in agents.json") + return name, AGENTS["agents"][name] + + +def _paired(model): + """The agent option's agent and its entry, for a model it is published with only.""" + name, spec = _agent() + if model not in spec["models"]: + others = [n for n, s in AGENTS["agents"].items() if model in s["models"]] + raise RuntimeError(f"{model} is not one of the {name} agent's models ({', '.join(spec['models'])}): " + + (f"choose the agent {others[0]} for it, or one of them" if others else "choose one of them")) + return name, spec + + +def _fill(value, slots): + for k, v in slots.items(): + value = value.replace("{" + k + "}", v) + return value + + +def _result(dir, domain, eid): + found = glob.glob(os.path.join(dir, RESULTS, "**", domain, eid, "result.txt"), recursive=True) + return os.path.dirname(found[0]) if found else None + + +def _traj(where, name="traj.jsonl"): + try: + with open(os.path.join(where, name), encoding="utf-8", errors="replace") as f: + lines = f.read().split("\n") + except OSError: + return [] + out = [] + for ln in lines: + with contextlib.suppress(ValueError): + out.append(json.loads(ln)) + return out + + +def _command(dir, model, base_url, key, vm): + """M3's thinking option is M3_THINKING_MODE (empty: the documented default); Muse Spark's reasoning effort is in its argv. + A dotenv file the script loads overrides no variable already set.""" + name, spec = _paired(model) + meta = os.path.join(dir, "test_unit.json") + slots = {"model": model, "base_url": base_url, "key": key, "vm": vm, "up": UP, "meta": meta, + "results": os.path.join(dir, RESULTS), "examples": EXAMPLES, + "base": _base(base_url), + "max_steps": os.environ.get("NULLIUS_OPTION_MAX_STEPS", "100")} + common, drop = list(AGENTS["common"]), spec.get("drop", []) + for flag in drop: + i = common.index(flag) + del common[i:i + (1 if flag == "--headless" else 2)] + argv = [sys.executable, os.path.join(UP, spec["script"])] + [_fill(a, slots) for a in + common + spec["argv"] + spec.get("model_argv", {}).get(model, [])] + extra = {k: _fill(v, slots) for k, v in spec["env"].items()} + if name == "m3": + thinking = os.environ.get("NULLIUS_OPTION_THINKING", "off") + extra.update(M3_THINKING_MODE="adaptive" if thinking == "adaptive" else "", M3_THINKING_BUDGET="") + return spec, meta, argv, extra + + +@contextlib.contextmanager +def _set_up(): + """What fails before the upstream's process starts is the unit's environment's.""" + try: + yield + except (RuntimeError, OSError, LookupError, ValueError, subprocess.SubprocessError, tarfile.TarError, EOFError, + zlib.error) as e: + raise e if isinstance(e, EnvironmentSetupError) else EnvironmentSetupError(f"{type(e).__name__}: {e}") from None + + +def _services(held, dir, domain, eid, until=None): + closed = _Waiting(until) + ports = held.enter_context(_published(eid, closed)) + held.enter_context(websites.held(eid, websites.needed(eid), closed)) + held.enter_context(_gitlab(eid, dir, closed)) + return dict(ports, **held.enter_context(proxy.held(dir, _task(dir, domain, eid)["proxy"], _disk))) + + +@_letting_go +def answer(unit, dir, setup, model, base_url, key, judge=None, **_): + if setup != "original": + raise RuntimeError(f"unknown setup {setup}: OSWorld2 runs original and its null control") + until = time.time() + _turn_seconds() - EVALUATE_SECONDS - CLOSE_SECONDS + domain, eid = _unit(unit) + with contextlib.ExitStack() as held: + with _set_up(): + _fresh(dir) + script = os.path.basename(_agent()[1]["script"]) + log = os.path.join(dir, script.replace(".py", ".log")) + graded = _graded(eid, judge) + _asks_user(eid, *_paired(model)) + vm = _vm(dir, domain, eid) + spec, meta, argv, extra = _command(dir, model, base_url, key, vm) + extra.update(graded, OSWORLD2_STOP=_stop_file(dir)) + with open(meta, "w", encoding="utf-8") as f: + json.dump({domain: [eid]}, f) + services = _services(held, dir, domain, eid, until) + assets = _assets(dir, eid) + _run(argv, dir, _env(eid, assets, **extra, **services), log, script, stagger=True, until=until) + where = _result(dir, domain, eid) + if where is None: + kind, said = _tail(log) + raise kind(f"{script} wrote no result.txt: {said}") + steps = _traj(where) + last = steps[-1].get("action") if steps else "" + return {"answer": last if isinstance(last, str) else json.dumps(last), "model": model} + + +def _fresh(dir): + while _OWN: + os.close(_OWN.pop()) + os.makedirs(dir, exist_ok=True) + _OWN.append(websites.opened(os.path.join(dir, "unit.lck"))) + t0 = time.time() + while True: + try: + fcntl.flock(_OWN[0], fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + if CLOSED.wait(1) or time.time() - t0 > HELD_SECONDS: + raise EnvironmentSetupError("an earlier run of this unit, or its reaper, still holds its folder") + ns = time.time_ns() + for name in [RESULTS] + [os.path.basename(p) for p in glob.glob(os.path.join(glob.escape(dir), "*.log"))]: + if os.path.lexists(os.path.join(dir, name)): + os.replace(os.path.join(dir, name), os.path.join(dir, f"{name}.{ns}")) + + +def _turn_seconds(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f)["setups"]["original"]["turn_seconds"] + + +def _score_seconds(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return (json.load(f).get("limits") or {}).get("score_seconds") or 600 + + +def _acted(dir): + for path in glob.glob(os.path.join(glob.escape(os.path.dirname(_stop_file(dir))), "*.jsonl")): + if "-" not in os.path.basename(path): + for call in _traj(os.path.dirname(path), os.path.basename(path)): + if isinstance(call, dict) and not call.get("side"): + return True + return False + + +def _stop_file(dir): + return os.path.join(os.path.dirname(os.path.abspath(dir)), ".work", "relay", "stop") + + +def _said(path): + with open(path, encoding="utf-8") as f: + return [ln.strip() for ln in f.read().splitlines() if ln.strip()] + + +def _stopped(dir): + """The stop that ended the episode's step (sitecustomize's .taken, the stop file if taken before it was written), + its first word and every reason's kind.""" + try: + said = _said(_stop_file(dir) + ".taken") + except OSError: + return {} + with contextlib.suppress(OSError): + said = said or _said(_stop_file(dir)) + return {"stopped": said[0] if said else "stop", "stopped_by": list(dict.fromkeys( + next((why for start, why in STOPPED_BY if ln.startswith(start)), "other") for ln in said))} + + +def _scored(where, how): + """The safety checks' own words are the task's, so they stay in the trial's result.json.""" + try: + with open(os.path.join(where, "result.txt"), encoding="utf-8") as f: + value = float(f.read().strip()) + except (OSError, ValueError): + raise VerifierError("result.txt could not be read as a number") from None + if not -1e-9 <= value <= 1 + 1e-9: + raise VerifierError(f"result.txt holds {value}, not a score from 0 to 1") + steps = _traj(where) + detail = {"result": value, "by": how, "steps": len(steps), + "last_action": steps[-1].get("action") if steps else None} + try: + with open(os.path.join(where, "result.json"), encoding="utf-8") as f: + evaluated = json.load(f) + except (OSError, ValueError): + evaluated = {} + if isinstance(evaluated, dict): + if "score" in evaluated: + detail["score"] = evaluated["score"] + safety = evaluated.get("safety") + if isinstance(safety, dict): + checks = [c for c in safety.get("checks") or [] if isinstance(c, dict)] + detail["safety"] = {"total_penalty": safety.get("total_penalty"), "checks": len(checks), + "failed": sum(1 for c in checks if c.get("result") == "FAIL")} + return {"reward": value, "detail": detail} + + +def _set_up_done(log): + try: + with open(log, encoding="utf-8", errors="replace") as f: + return any(SET_UP_DONE.match(COLOUR.sub("", ln).rstrip("\n")) for ln in f) + except OSError: + return False + + +def _setup_errors(*logs): + """The upstream's setup errors it logged and went on from, from the last attempt to set the task up to the last step, + or to that setup's end with no step: none retried, none of the evaluation's own setup.""" + for log in logs: + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip("\n") for ln in f] + except OSError: + continue + start = max((i for i, ln in enumerate(lines) if SET_UP.match(ln)), default=None) + if start is not None: + end = max((i for i, ln in enumerate(lines) if i > start and STEPPED.match(ln)), default=None) or next( + (i for i, ln in enumerate(lines) if i > start and SET_UP_DONE.match(ln)), len(lines)) + errors = [m for m in map(SETUP_ERROR.match, lines[start:end]) if m and not SETUP_RETRY.search(m.group(1))] + return {"setup_errors": len(errors)} if errors else {} + return {} + + +def _grader_cause(said): + """A grader error record's HTTP status and its class from GRADER_CLASSES, never its words, which can quote the task.""" + said = said.strip() + status, name = GRADER_STATUS.search(said), GRADER_NAME.match(said) + module = name.group(1).rstrip(".") if name else "" + module = module[:-len("._exceptions")] if module.endswith("._exceptions") else module + known = name and name.group(2) in GRADER_CLASSES.get(module, ()) + found = ([f"{module}.{name.group(2)}" if module else name.group(2)] if known else []) \ + + ([f"HTTP {status.group(1)}"] if status else []) + return ", ".join(found) or "no HTTP status or known error class in its record" + + +def _grader_failed(*logs, refused=0): + """The upstream counts a grader error as a failed comparison; only its own ERROR record counts (_grader_cause). + Each of the ``refused`` calls of the agent's files accounts for one 400 or 422 record.""" + for log in logs: + left = refused + try: + with open(log, encoding="utf-8", errors="replace") as f: + for line in f: + m = GRADER_RECORD.match(COLOUR.sub("", line).rstrip("\n")) + status = GRADER_STATUS.search(m.group(2)) if m and left else None + if status and status.group(1) in ("400", "413", "422") and m.group(1) in GRADER_FAILED: + left -= 1 + elif m and m.group(1) in GRADER_FAILED: + raise VerifierError(f"the LLM grader failed in {m.group(1)[len('Error in '):-2]} " + f"({_grader_cause(m.group(2))}), and the upstream counted its " + "comparison as failed; see the unit's judge route") + except OSError: + continue + + +def _unread(value, *logs, retried=False): + """A read of the VM or a site that the last episode's evaluation, any of its phases, gave up on unanswered, by what and how only.""" + for log in logs: + try: + with open(log, encoding="utf-8", errors="replace") as f: + lines = [COLOUR.sub("", ln).rstrip("\n") for ln in f] + except OSError: + continue + marks = [(i, m) for i, m in enumerate(map(RUNNER.match, lines)) if m] + ends = [k for k, (_, m) in enumerate(marks) if not m["step"] and not m["phase"]] + windows, end = [], marks[ends[-1]][0] if ends else None + for i, m in reversed(marks[ends[-2] + 1 if len(ends) > 1 else 0:ends[-1]] if ends else []): + if m["phase"]: + end = i + elif m["step"] and end is not None: + windows.append((i, end)) + end = None + if m["n"] in (None, "1"): + break + site, unread = None, None + for a, b in windows: + last = None + for ln in lines[a:b]: + state, said = STATE.match(ln), CONTROLLER.match(ln) + read = READ.match(said.group(1)) if said else None + status = SITE_STATUS.search(state.group(2)) if state else None + if state and not site and not retried and (not status or status.group(1) in GATEWAY): + site = (f"a site's {'state' if state.group(1) == 'fetch state' else 'file'}", f"HTTP {status.group(1)}" + if status else "timed out" if "timed out" in state.group(2) else "no answer") + elif read and (read["lost"] or read["said"]): + last = (read["lost"] or read["said"], read["status"]) + elif read and last and read["again"] == last[0]: + continue + elif read and last and read["gave"] == last[0] and not last[1]: + unread = unread or (f"the VM's {last[0]}", "a broken connection") + elif said: + last = None + unread = unread or site + if unread: + raise VerifierError(f"the evaluation could not read {unread[0]} ({unread[1]}), and the upstream scored " + f"the task {value} without it; see the unit's {os.path.basename(log)}") + + +def _refusal(call): + """A judge call refused for the agent's file or its size, by the images its request carried (the relay's ``images``), + never by the provider's words.""" + images = [i for i in call.get("images") or () if isinstance(i, dict)] + if call.get("status") in (400, 413, 422) and any(type(i.get("bytes")) is int and i["bytes"] > IMAGE_BYTES for i in images): + return "agent file size" + if call.get("status") in (400, 422) and any(i.get("is") not in IMAGE_TYPES for i in images): + return "agent file" + return None + + +def _grader_refused(dir, judge): + """A judge-route call not 200 counts unless a later call with the same request reaches 200; its status alone is said. + Some task classes catch the grader's error and score it themselves. The class of each call that refused the agent's + files.""" + route = JUDGE_ROUTE.search(str(judge.get("base_url") or "")) if isinstance(judge, dict) else None + if not route: + return [] + try: + with open(os.path.join(os.path.dirname(os.path.abspath(dir)), ".work", "relay", route.group(1) + ".jsonl"), + encoding="utf-8") as f: + lines = f.read().splitlines() + except OSError: + return [] + calls = [] + for ln in lines: + try: + call = json.loads(ln) + except ValueError: + continue + if isinstance(call, dict): + calls.append(call) + if all(isinstance(c.get("t0"), (int, float)) for c in calls): + calls.sort(key=lambda c: c["t0"]) + refused, failed = [], None + for i, call in enumerate(calls): + if call.get("status") == 200: + continue + asked, recovered = call.get("request"), False + for retry in calls[i + 1:]: + if not isinstance(asked, dict) or not asked.get("sha256") or retry.get("request") != asked: + break + if retry.get("status") == 200: + recovered = True + break + why = None if recovered else _refusal(call) + if not recovered and not why: + failed = failed or call + refused += [why] if why else [] + if failed: + raise VerifierError(f"the LLM grader's endpoint answered {failed.get('status')}, and the task was " + "scored without it; see the unit's judge route") + return refused + + +def score(*a, **kw): + try: + return _score(*a, **kw) + finally: + while _OWN: + os.close(_OWN.pop()) + + +def _score(unit, dir, transcript=None, answer=None, setup=None, judge=None, **_): + domain, eid = _unit(unit) + if setup == "original": + where = _result(dir, domain, eid) + if where is None: + raise VerifierError("no result.txt: the upstream's runner did not finish the task") + script = os.path.basename(_agent()[1]["script"]) + stopped = _stopped(dir) + if stopped and not _acted(dir): + raise (EnvironmentSetupError if stopped["stopped"] == "turn limit" else CancelledError)( + f"the unit was stopped ({stopped['stopped']}) before the agent's first step") + refused = _grader_refused(dir, judge) + logs = os.path.join(where, "eval.log"), os.path.join(dir, script.replace(".py", ".log")) + _grader_failed(*logs, refused=len(refused)) + scored = _scored(where, script + ", DesktopEnv.evaluate") + _unread(scored["reward"], *logs, retried=eid in RETRIES_SITE_READS) + return dict(scored, **stopped, **({"grader_refused": sorted(set(refused))} if refused else {}), + **_setup_errors(*logs)) + if setup is not None: + raise RuntimeError(f"unknown setup {setup}: OSWorld2 runs original and its null control") + if answer: + raise RuntimeError("OSWorld2 has no agent side yet: agent.computer/v1 has no kit") + until = time.time() + _score_seconds() - 2 * CLOSE_SECONDS + where = os.path.join(dir, RESULTS, "nop", domain, eid) + failed = os.path.join(where, SETUP_FAILED) + with contextlib.ExitStack() as held: + with _set_up(): + graded = _graded(eid, judge) + vm = _vm(dir, domain, eid) + os.makedirs(where, exist_ok=True) + if os.path.lexists(failed): + os.remove(failed) + services = _services(held, dir, domain, eid, until) + assets = _assets(dir, eid) + _run([sys.executable, NOP, vm, PASSWORD, EXAMPLES, domain, eid, where], + dir, _env(eid, assets, **graded, **services), os.path.join(dir, "nop.log"), "the null control's run", + stagger=True, failed=failed, until=until, grace=CLOSE_SECONDS) + if not os.path.exists(os.path.join(where, "result.txt")): + kind, said = _tail(os.path.join(dir, "nop.log")) + raise kind(f"the null control's run wrote no result.txt: {said}") + refused = _grader_refused(dir, judge) + _grader_failed(os.path.join(dir, "nop.log"), refused=len(refused)) + scored = _scored(where, "nop.py, DesktopEnv.evaluate") + _unread(scored["reward"], os.path.join(dir, "nop.log"), retried=eid in RETRIES_SITE_READS) + return dict(scored, **({"grader_refused": sorted(set(refused))} if refused else {}), **_setup_errors(os.path.join(dir, "nop.log"))) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--reap"]: + reap(int(sys.argv[2]), sys.argv[3], sys.argv[4:5] == ["vms"], sys.stdin.read().split()) + sys.exit(0) + if sys.argv[1:2] == ["--ward"]: + gitlab_server.ward(int(sys.argv[2]), sys.argv[3]) + sys.exit(0) + HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score} + threading.Thread(target=_read, daemon=True).start() + while True: + line = LINES.get() + if line is None: + break + try: + req = json.loads(line) + out = HANDLERS[req.pop("op")](**req) + except Exception as e: + out = {"error": f"{type(e).__name__}: {e}"} + print(json.dumps(out), flush=True) diff --git a/app/nullius/catalogue/benchmarks/osworld2/gitlab.Dockerfile b/app/nullius/catalogue/benchmarks/osworld2/gitlab.Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml b/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml new file mode 100644 index 0000000..0b531ed --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/gitlab.compose.yml @@ -0,0 +1,94 @@ +name: osworld2-gitlab + +x-logging: &logs + driver: json-file + options: + max-size: 10m + max-file: "3" + +services: + gitlab: + image: gitlab/gitlab-ce@sha256:9c17686c3cd1b72f52282543d2dcf7c96add29b4698ec367bfb1455be8f13ce0 + container_name: !reset null + restart: "no" + ports: !reset [] + networks: + web: + aliases: + - nullius-osworld2-gitlab + labels: + caddy: "http://${OSWORLD2_GITLAB_HOST:?as gitlab_server.py sets it}, http://*.${OSWORLD2_GITLAB_HOST:?as gitlab_server.py sets it}" + caddy.reverse_proxy: "nullius-osworld2-gitlab:80" + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + volumes: + - ${OSWORLD2_GITLAB_RB:?as gitlab_server.py sets it}:/etc/gitlab/gitlab.rb:ro + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_GITLAB_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_GITLAB_IOPS:?as gitlab_server.py sets it} + logging: *logs + gitlab-init-token: + profiles: + - upstream-init + volumes: !override [] + entrypoint: !override ["/bin/false"] + command: !override [] + gitlab-runner: + image: gitlab/gitlab-runner@sha256:9d3b2b8be462e18176b82eb40a5f54a5ee6eaa2b4f46dcc362c4801aef90030f + container_name: !reset null + labels: + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + restart: "no" + depends_on: + - dind + networks: + - ci + volumes: !override + - gitlab_runner_config:/etc/gitlab-runner + - dind_certs_client:/certs/client:ro + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_RUNNER_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_RUNNER_IOPS:?as gitlab_server.py sets it} + logging: *logs + dind: + image: docker@sha256:3ef33f2e220b79ed3ef3b99d81746f06f306cd6340e2cb7331d17ae996e74cb6 + labels: + nullius.osworld2.share: ${OSWORLD2_GITLAB_SHARE:-stack} + restart: "no" + privileged: true + environment: + DOCKER_TLS_CERTDIR: /certs + networks: + ci: + aliases: + - docker + volumes: + - dind_certs_ca:/certs/ca + - dind_certs_client:/certs/client + - dind_data:/var/lib/docker + blkio_config: + device_write_bps: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_DIND_BPS:?as gitlab_server.py sets it} + device_write_iops: + - path: ${OSWORLD2_GITLAB_DISK:?the disk under Docker's root, as gitlab_server.py sets it} + rate: ${OSWORLD2_GITLAB_DIND_IOPS:?as gitlab_server.py sets it} + logging: *logs + +networks: + web: + name: ${OSWORLD2_GITLAB_INGRESS:?the mock websites' Caddy network, as gitlab_server.py sets it} + external: true + ci: {} + +volumes: + dind_certs_ca: + dind_certs_client: + dind_data: diff --git a/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py b/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py new file mode 100644 index 0000000..c1ba91e --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/gitlab_server.py @@ -0,0 +1,592 @@ +"""OSWorld2's GitLab: Task-Web/gitlab at COMMIT, for the tasks of TASKS. + +GitLab is that repository's compose project with gitlab.compose.yml over it. +It publishes no port: the mock websites' Caddy serves it as gitlab.. +init makes the root token, registers the runner and keeps the volumes as the +snapshot. A unit of TASKS holds LOCK and starts GitLab from the snapshot; a +warden in a session of its own removes it once the unit lets go or its driver +dies. + +python3 gitlab_server.py configure STACK SUFFIX --write-mbps N --write-iops N +python3 gitlab_server.py init STACK | up STACK | down STACK | check HOST [--runner] +python3 gitlab_server.py compose STACK ARGS...""" +import argparse +import contextlib +import fcntl +import json +import os +import re +import shlex +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +import websites + +HERE = os.path.dirname(os.path.abspath(__file__)) +OVERRIDE = os.path.join(HERE, "gitlab.compose.yml") +COMMIT = "8655d651722f4254e59e813de9f68a6732ea525c" +PROJECT = "osworld2-gitlab" +OPTION = "NULLIUS_OPTION_GITLAB" +SHARE_OPTION = "NULLIUS_OPTION_GITLAB_WRITE" +VM_WRITE = ("NULLIUS_OPTION_WRITE_MBPS", "NULLIUS_OPTION_WRITE_IOPS") +TOKEN = "GITLAB_PRIVATE_TOKEN" +TASKS = frozenset(("026", "041")) +RUNNER_TASKS = frozenset(("041",)) +LOCK = "/tmp/nullius-osworld2-gitlab.lck" +HOST = re.compile(r"[a-z0-9](?:[a-z0-9.-]{0,56}[a-z0-9])?(?::[1-9][0-9]{0,4})?") +SHARE = re.compile(r"[1-9][0-9]{0,5}:[1-9][0-9]{0,5}") +ENV_FILE = ".osworld2.env" +COMPOSE_FILE = ".osworld2.compose.yml" +RB_FILE = ".osworld2.gitlab.rb" +RB = """puma['worker_processes'] = 2 +sidekiq['concurrency'] = 5 +prometheus_monitoring['enable'] = false +""" +SHARES = (("GITLAB", 6), ("RUNNER", 1), ("DIND", 1)) +LIMITS = tuple(f"OSWORLD2_GITLAB_{s}_{k}" for s, _ in SHARES for k in ("BPS", "IOPS")) +SETTINGS = ("GITLAB_URL", "GITLAB_PRIVATE_TOKEN", "OSWORLD2_GITLAB_HOST", "OSWORLD2_GITLAB_INGRESS", + "OSWORLD2_GITLAB_DISK", "OSWORLD2_GITLAB_RB", "OSWORLD2_GITLAB_SHARE") + LIMITS +UNUSED = "unused" +SNAPSHOT = "nullius-osworld2-gitlab-snapshot" +STACK_FILE = "stack" +GITLAB_IMAGE = re.compile(r"^\s+image: (gitlab/gitlab-ce@sha256:[0-9a-f]{64})$", re.M) +SHARE_LABEL = "nullius.osworld2.share" +ARCHIVE = "gitlab.tar.gz" +VOLUMES = ("gitlab_config", "gitlab_data", "gitlab_runner_config") +MARK = "gitlab.stack" +WARD_LOG = "gitlab-warden.log" +WARD_SWEEP = 20 +COPY_LABEL = "nullius.osworld2.gitlab.copy" +UNIT_LIMITS = "gitlab.limits" +JOB_IMAGE = "alpine:3.20" +RUNNER_NAME = "osworld-docker-runner" +DOCKER_HOST = "tcp://docker:2376" +MIN_TOKEN = 20 +HTTP_SECONDS = 15 +START_SECONDS = 3600 +RUNNER_SECONDS = 300 +COPY_SECONDS = 3600 +POLL = 10 +PROXIES = {} +INIT = "make GitLab once (python3 gitlab_server.py init STACK, from this benchmark's folder)" + + +class GitLabError(RuntimeError): + pass + + +def url(environ=os.environ): + """GitLab's URL from the run's GitLab option, its host; None for none.""" + value = environ.get(OPTION, "none") + if value == "none": + return None + if not HOST.fullmatch(value): + raise GitLabError(f"the GitLab option {value!r} is not GitLab's host: give the name it answers at " + f"(such as gitlab.{websites._site().GATEWAY}.nip.io), or none") + return websites.SCHEME + value + + +def needed(eid, environ=os.environ): + """A task of TASKS without GitLab's host and token is refused, since its class cannot even load.""" + value = url(environ) + if eid not in TASKS: + return None + if not value or not environ.get(TOKEN): + raise GitLabError(f"task {eid} needs GitLab and this run gives it no " + + ("host: set the GitLab option" if not value else f"token: set the licence {TOKEN}") + + f" ({websites.SETUP_DOC})") + return value + + +def env(eid, environ=os.environ): + """The variables the upstream reads GitLab by, for a task of TASKS; its token is in the driver's own.""" + value = url(environ) if eid in TASKS else None + return {"GITLAB_URL": value} if value else {} + + +def share(environ=os.environ): + """The run's GitLab write share, (MB/s, IOPS); None for none, refused while the VMs' writes are limited.""" + value = environ.get(SHARE_OPTION, "none") + found = SHARE.fullmatch(value) + if value != "none" and not found: + raise GitLabError(f"the GitLab write share {value!r} is not MB/s:IOPS, or none") + if not found and any(environ.get(n, "none") != "none" for n in VM_WRITE): + raise GitLabError("the VMs' writes are limited and GitLab has no write share of its own: take a slot for it " + "beside the VM's and give it as the GitLab write share option, MB/s:IOPS") + given = tuple(int(x) for x in value.split(":")) if found else None + if given: + allocation(*given) + return given + + +def stack(): + """The stack's folder, as init wrote it beside the snapshot's archive.""" + if not _snapshot(): + raise GitLabError(f"no snapshot {SNAPSHOT} on this machine: {INIT}") + image = GITLAB_IMAGE.search(_override()) + said = _docker("run", "--rm", "--network", "none", "--pull", "never", "-v", f"{SNAPSHOT}:/snapshot:ro", + "--entrypoint", "cat", image.group(1) if image else "", f"/snapshot/{STACK_FILE}") + found = said.stdout.strip() if said.returncode == 0 else "" + if not os.path.isabs(found): + raise GitLabError(f"the snapshot {SNAPSHOT} names no stack's folder: {INIT}") + return found + + +def prepared(environ=os.environ): + """What a unit's GitLab needs before its VM starts: the snapshot, the stack's settings, the write share.""" + where = stack() + served = _settings(where).get("GITLAB_URL") + if url(environ) != served: + raise GitLabError(f"the GitLab option asks for {url(environ)} and the stack at {where} serves {served}: " + "set the GitLab option to the stack's host") + share(environ) + return where + + +def _request(path, value, token=None, data=None, method=None): + """``path`` of GitLab at ``value``, as (status, body); no answer is a GitLabError.""" + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + headers = {"User-Agent": "nullius-osworld2", **({"PRIVATE-TOKEN": token} if token else {})} + body = urllib.parse.urlencode(data).encode() if data is not None else None + try: + with opener.open(urllib.request.Request(value + path, data=body, headers=headers, method=method), + timeout=HTTP_SECONDS) as r: + return r.status, r.read(1 << 20) + except urllib.error.HTTPError as e: + e.close() + return e.code, b"" + except (urllib.error.URLError, OSError, ValueError) as e: + reason = getattr(e, "reason", e) + raise GitLabError(f"{value}{path} did not answer ({type(reason).__name__}: {reason})") from None + + +def _json(said, what): + try: + return json.loads(said[1]) + except ValueError: + raise GitLabError(f"{what} answered no JSON") from None + + +def ready(value, token, runner=False): + """GitLab's sign-in page, the root token as an administrator's and, with ``runner``, an online runner.""" + host = urllib.parse.urlsplit(value).hostname + if not websites._resolves(host): + raise GitLabError(f"{host} does not resolve: the GitLab option must be the host GitLab answers at") + if not _signed_in(value): + raise GitLabError(f"{value}/users/sign_in answered no page") + said = _request("/api/v4/user", value, token) + if said[0] != 200 or _json(said, f"{value}/api/v4/user").get("is_admin") is not True: + raise GitLabError(f"{value}/api/v4/user answered {said[0]} for the token {TOKEN}, or not as an " + "administrator's: give the token GitLab was made with (python3 gitlab_server.py init)") + if runner: + said = _request("/api/v4/runners/all?type=instance_type&status=online", value, token) + if said[0] != 200 or not _json(said, f"{value}/api/v4/runners/all"): + raise GitLabError(f"GitLab at {value} has no runner online, and task 041 deploys with one") + return value + + +@contextlib.contextmanager +def held(value, closed): + """With GitLab's URL, GitLab alone, and the mock websites, which serve it, kept from a reset.""" + if not value: + yield + return + with websites.locked(websites.SITES_LOCK, fcntl.LOCK_SH, closed, "the stack was being reset or smoked"), \ + websites.locked(LOCK, fcntl.LOCK_EX, closed, "another unit's GitLab task ran"): + yield + + +@contextlib.contextmanager +def _alone(): + with websites.locked(websites.SITES_LOCK, fcntl.LOCK_SH, None, "the mock websites are being reset or smoked: " + "start or remove GitLab by hand when they are not"), \ + websites.locked(LOCK, fcntl.LOCK_EX, None, "a unit's GitLab task runs: start or remove GitLab by hand " + "when no task that needs it runs"): + yield + + +def unit(where, dir, environ=os.environ): + """The unit's mark for its reaper and, with a write share, the limits its GitLab takes from it.""" + given = share(environ) + limits = os.path.join(dir, UNIT_LIMITS) if given else None + if limits: + lines = {f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) for s, v in allocation(*given).items() + for i, k in enumerate(("BPS", "IOPS"))} + websites._write(limits, "".join(f"{k}={v}\n" for k, v in lines.items()) + "OSWORLD2_GITLAB_SHARE=unit\n") + websites._write(os.path.join(dir, MARK), json.dumps({"stack": where, "limits": limits})) + return limits + + +def reap(dir): + """GitLab removed for a unit that started it, and again WARD_SWEEP seconds later; nothing without its mark.""" + mark = os.path.join(dir, MARK) + try: + with open(mark, encoding="utf-8") as f: + said = json.load(f) + except (OSError, ValueError): + return False + down(said["stack"], said.get("limits")) + time.sleep(WARD_SWEEP) + if _running(): + down(said["stack"], said.get("limits")) + os.remove(mark) + return True + + +def ward(fd, dir): + """The warden: once ``fd`` closes, GitLab is removed, then the locks let go; failures go to WARD_LOG.""" + try: + while os.read(fd, 4096): + pass + except OSError: + pass + try: + reap(dir) + except (GitLabError, websites.WebsiteError, OSError, ValueError, KeyError) as e: + with open(os.path.join(dir, WARD_LOG), "a", encoding="utf-8") as f: + f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {type(e).__name__}: {e}\n") + + +def _running(): + said = _docker("ps", "-aq", "--filter", f"label=com.docker.compose.project={PROJECT}") + return said.returncode == 0 and bool(said.stdout.split()) + + +def _docker(*args, seconds=websites.DOCKER_SECONDS, **kw): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, + env=kw.pop("env", _clean()), stdin=subprocess.DEVNULL, **kw) + except subprocess.TimeoutExpired: + raise GitLabError(f"docker {args[0]} got no answer in {seconds} s") from None + except OSError: + raise GitLabError("no docker on this machine's PATH: GitLab runs on Docker") from None + + +def _clean(): + """Without the project's settings or the token, so compose reads them from its environment files alone.""" + return {k: v for k, v in os.environ.items() if k not in SETTINGS} + + +def _done(said, what): + if said.returncode != 0: + last = ((said.stderr or said.stdout or "").strip().splitlines() or ["no reason"])[-1] + raise GitLabError(f"{what} failed: {last[:300]}") + return said + + +def _pinned(stack): + """``stack`` is Task-Web/gitlab at COMMIT, nothing tracked changed.""" + def git(*args): + try: + return subprocess.run(["git", "-C", stack, *args], capture_output=True, text=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as e: + raise GitLabError(f"git could not read {stack}: {type(e).__name__}") from None + if git("rev-parse", "HEAD").stdout.strip() != COMMIT: + raise GitLabError(f"{stack} is not Task-Web/gitlab at {COMMIT}: clone it at that commit") + changed = git("status", "--porcelain", "--untracked-files=no") + if changed.returncode != 0 or changed.stdout.strip(): + raise GitLabError(f"{stack} has changes to its files: GitLab runs as published") + return os.path.abspath(stack) + + +def _beside(stack, name): + return os.path.abspath(stack).rstrip(os.sep) + name + + +def _read(path): + with open(path, encoding="utf-8") as f: + return dict(ln.rstrip("\n").partition("=")[::2] for ln in f if "=" in ln) + + +def _settings(stack, limits=None): + path = _beside(stack, ENV_FILE) + try: + return dict(_read(path), **(_read(limits) if limits else {})) + except OSError as e: + raise GitLabError(f"no settings of GitLab at {path} ({type(e).__name__}): run python3 gitlab_server.py " + "configure first") from None + + +def allocation(mbps, iops): + """Each service's limit by its share, rounded down, so all of them stay within the project's.""" + if mbps < 1 or iops < 1: + raise GitLabError("GitLab's write share is whole numbers above 0") + total = sum(w for _, w in SHARES) + out = {s: (mbps * websites.MB * w // total, iops * w // total) for s, w in SHARES} + if not all(v for pair in out.values() for v in pair): + raise GitLabError(f"a write share of {iops} IOPS leaves a service none: give GitLab at least {total}") + return out + + +def configure(stack, suffix, mbps, iops): + stack = _pinned(stack) + if not websites.SUFFIX.fullmatch(suffix): + raise GitLabError(f"{suffix!r} is not a host suffix") + shares = allocation(mbps, iops) + websites._network() + rb = _beside(stack, RB_FILE) + websites._write(rb, RB) + settings = {"GITLAB_URL": f"{websites.SCHEME}gitlab.{suffix}", "GITLAB_PRIVATE_TOKEN": UNUSED, + "OSWORLD2_GITLAB_HOST": f"gitlab.{suffix}", "OSWORLD2_GITLAB_INGRESS": f"{websites.PROJECT}_web", + "OSWORLD2_GITLAB_DISK": websites._root_disk()[1], "OSWORLD2_GITLAB_RB": rb, + "OSWORLD2_GITLAB_SHARE": "stack", + **{f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) + for s, v in shares.items() for i, k in enumerate(("BPS", "IOPS"))}} + path = _beside(stack, ENV_FILE) + websites._write(path, "".join(f"{k}={settings[k]}\n" for k in SETTINGS)) + return path, settings + + +def _override(): + try: + with open(OVERRIDE, encoding="utf-8") as f: + return f.read() + except OSError as e: + raise GitLabError(f"no {OVERRIDE} ({type(e).__name__}): this benchmark's folder is incomplete; " + "install Nullius again") from None + + +def _argv(stack, args, limits=None): + copy, text = _beside(stack, COMPOSE_FILE), _override() + try: + with open(copy, encoding="utf-8") as f: + same = f.read() == text + except OSError: + same = False + if not same: + websites._write(copy, text) + return ["compose", "-p", PROJECT, "--project-directory", stack, "-f", os.path.join(stack, "docker-compose.yml"), + "-f", copy, "--env-file", _beside(stack, ENV_FILE), *(["--env-file", limits] if limits else []), *args] + + +def compose(stack, args, capture=True, seconds=websites.RESET_SECONDS, limits=None): + stack = _pinned(stack) + _settings(stack, limits) + websites._compose_version() + if capture: + return _docker(*_argv(stack, args, limits), seconds=seconds) + return subprocess.run(["docker", *_argv(stack, args, limits)], env=_clean(), pass_fds=websites.held_fds()).returncode + + +def _token(): + value = os.environ.get(TOKEN, "") + if len(value) < MIN_TOKEN or not value.isprintable() or value.strip() != value: + raise GitLabError(f"give GitLab's root token as {TOKEN} in this command's environment, at least " + f"{MIN_TOKEN} printable characters") + return value + + +def _container(stack, service): + said = _done(compose(stack, ["ps", "-a", "-q", service]), f"finding {service}") + found = said.stdout.split() + if len(found) != 1: + raise GitLabError(f"no one container of {service} in {PROJECT}") + return found[0] + + +def _until(seconds, what, check, closed=None): + t0, last = time.time(), "no answer" + while True: + try: + if check(): + return round(time.time() - t0) + except GitLabError as e: + last = str(e)[:200] + if time.time() - t0 >= seconds: + raise GitLabError(f"{what} did not come up in {seconds} s ({last})") + if closed is None: + time.sleep(POLL) + elif closed.wait(POLL): + raise GitLabError(f"{what} was not up when the unit's time ran out") + + +def _signed_in(value): + """Caddy answers a name it serves no site for with an empty page.""" + status, body = _request("/users/sign_in", value) + return status == 200 and bool(body.strip()) + + +def _upstream_script(stack): + """The upstream's gitlab-init-token script, from its own compose file: the override disarms that service.""" + said = _done(_docker("compose", "-p", PROJECT, "--project-directory", stack, "-f", + os.path.join(stack, "docker-compose.yml"), "--env-file", _beside(stack, ENV_FILE), + "config", "--format", "json"), "reading the upstream's project") + return json.loads(said.stdout)["services"]["gitlab-init-token"]["command"][-1] + + +def _make_token(stack): + """The upstream's own token step, run with this machine's docker: no socket is mounted in a container.""" + script, name = _upstream_script(stack), _container(stack, "gitlab") + swaps = ((f'-e {TOKEN}="$${TOKEN}" gitlab ', f"-e {TOKEN} {name} "), + ("docker exec gitlab ", f"docker exec {name} ")) + for old, new in swaps: + if script.count(old) != 1: + raise GitLabError("the upstream's token step is not the one this file knows: check gitlab_server.py " + f"against Task-Web/gitlab at {COMMIT}") + script = script.replace(old, new) + done = subprocess.run(["sh", "-c", script.replace("$$", "$")], capture_output=True, text=True, + timeout=START_SECONDS, env=dict(_clean(), **{TOKEN: _token()}), stdin=subprocess.DEVNULL) + _done(done, "making the root token") + + +def _register(stack, value): + """The README's runner registration; a runner the container could not take is removed from GitLab again.""" + status, body = _request("/api/v4/user/runners", value, _token(), { + "runner_type": "instance_type", "description": RUNNER_NAME, "tag_list": "osworld,docker", + "run_untagged": "true", "locked": "false", "paused": "false"}) + if status != 201: + raise GitLabError(f"GitLab refused a runner ({status})") + made = json.loads(body) + runner = _container(stack, "gitlab-runner") + said = _docker("exec", "-e", "CI_SERVER_TOKEN", runner, "gitlab-runner", "register", "--non-interactive", + "--url", value, "--executor", "docker", "--docker-image", JOB_IMAGE, "--description", RUNNER_NAME, + "--docker-host", DOCKER_HOST, "--docker-tlsverify", "--docker-cert-path", "/certs/client", + env=dict(_clean(), CI_SERVER_TOKEN=made["token"])) + if said.returncode != 0: + _request(f"/api/v4/runners/{made['id']}", value, _token(), method="DELETE") + _done(said, "registering the runner") + return made["id"] + + +def _copy(stack, settings, tar, keep): + """``tar`` in a container of the project's, under GitLab's write limit; ``keep`` writes the snapshot.""" + image = _done(compose(stack, ["config", "--images"]), "reading the images").stdout.split() + gitlab = next(i for i in image if i.startswith("gitlab/gitlab-ce")) + disk = settings["OSWORLD2_GITLAB_DISK"] + mounts = [x for v in VOLUMES for x in ("-v", f"{PROJECT}_{v}:/v/{v}")] + return _done(_docker("run", "--rm", "--network", "none", "--label", f"com.docker.compose.project={PROJECT}", + "--label", f"{COPY_LABEL}=1", + "--label", f"{SHARE_LABEL}={settings.get('OSWORLD2_GITLAB_SHARE') or 'stack'}", + "--device-write-bps", f"{disk}:{settings['OSWORLD2_GITLAB_GITLAB_BPS']}", + "--device-write-iops", f"{disk}:{settings['OSWORLD2_GITLAB_GITLAB_IOPS']}", + *mounts, "-v", f"{SNAPSHOT}:/snapshot" + ("" if keep else ":ro"), + "--entrypoint", "sh", gitlab, "-c", tar, seconds=COPY_SECONDS), "copying the volumes") + + +def _volumes(): + said = _done(_docker("volume", "ls", "-q", "--filter", f"label=com.docker.compose.project={PROJECT}"), + "listing the volumes") + return said.stdout.split() + + +def _snapshot(): + return _docker("volume", "inspect", SNAPSHOT).returncode == 0 + + +def init(stack, say=print): + """GitLab made once from nothing: its first start, the root token, the runner; then kept as the snapshot.""" + stack = _pinned(stack) + settings = _settings(stack) + value, token = settings["GITLAB_URL"], _token() + if _volumes(): + raise GitLabError(f"{PROJECT} has volumes already: python3 gitlab_server.py down STACK first") + if _snapshot(): + raise GitLabError(f"the snapshot {SNAPSHOT} is there already: remove it (docker volume rm {SNAPSHOT}) to " + "make GitLab anew") + t0 = time.time() + _done(compose(stack, ["up", "-d", "--pull", "never", "gitlab"]), "starting GitLab") + say(json.dumps({"started": "gitlab", "up_s": _until(START_SECONDS, "GitLab", lambda: _signed_in(value))})) + _make_token(stack) + ready(value, token) + say(json.dumps({"token": "made", "s": round(time.time() - t0)})) + _done(compose(stack, ["up", "-d", "--pull", "never", "dind", "gitlab-runner"]), "starting the runner") + runner = _register(stack, value) + say(json.dumps({"runner": runner, "online_s": _until(RUNNER_SECONDS, "the runner", + lambda: _online(value, token, runner))})) + _done(compose(stack, ["stop"]), "stopping GitLab") + _done(_docker("volume", "create", SNAPSHOT), "making the snapshot's volume") + _copy(stack, settings, f"tar -C /v -czf /snapshot/{ARCHIVE} {' '.join(VOLUMES)} && " + f"printf %s {shlex.quote(stack)} > /snapshot/{STACK_FILE}", keep=True) + _done(compose(stack, ["down", "--volumes"]), "removing GitLab") + return {"snapshot": SNAPSHOT, "s": round(time.time() - t0)} + + +def _online(value, token, runner): + said = _request(f"/api/v4/runners/{runner}", value, token) + return said[0] == 200 and _json(said, "the runner").get("status") == "online" + + +def up(stack, limits=None, closed=None, say=None): + """GitLab anew from the snapshot, waited on until ready() and reached from the VMs' network.""" + stack = _pinned(stack) + settings = _settings(stack, limits) + value, token = settings["GITLAB_URL"], _token() + if not _snapshot(): + raise GitLabError(f"no snapshot {SNAPSHOT}: {INIT}") + t0 = time.time() + _done(compose(stack, ["down", "--volumes"], limits=limits), "removing GitLab") + _done(compose(stack, ["create", "--pull", "never"], limits=limits), "making GitLab's containers") + _copy(stack, settings, f"tar -C /v -xzf /snapshot/{ARCHIVE}", keep=False) + if say: + say(json.dumps({"restored_s": round(time.time() - t0)})) + _done(compose(stack, ["start"], limits=limits), "starting GitLab") + _until(START_SECONDS, "GitLab", lambda: ready(value, token, runner=True), closed) + _from_vms(settings["OSWORLD2_GITLAB_HOST"]) + return {"url": value, "up_s": round(time.time() - t0)} + + +def _from_vms(host): + """GitLab's sign-in page from a container on the VMs' network, by the mock websites' Caddy's busybox wget.""" + image = websites.CADDY.search(websites._override()).group(1) + said = _docker("run", "--rm", "--network", websites._network(), "--pull", "never", "--entrypoint", "wget", + image, "-q", "-O", "/dev/null", "-T", "10", f"http://{host}/users/sign_in") + if said.returncode != 0: + raise GitLabError(f"from the VMs' network, {host} did not answer: " + + ((said.stderr or "").strip().splitlines() or ["no reason"])[-1][:300]) + + +def down(stack, limits=None): + """The volumes' copy is no service of the project's, so it is removed by its label first.""" + said = _done(_docker("ps", "-aq", "--filter", f"label={COPY_LABEL}"), "listing the volumes' copy") + if said.stdout.split(): + _done(_docker("rm", "-f", *said.stdout.split()), "removing the volumes' copy") + _done(compose(stack, ["--profile", "upstream-init", "down", "--volumes", "--remove-orphans"], limits=limits), + "removing GitLab") + return {"removed": PROJECT} + + +def main(argv): + p = argparse.ArgumentParser(prog="gitlab_server.py", description="OSWorld2's GitLab") + sub = p.add_subparsers(dest="op", required=True) + c = sub.add_parser("configure") + c.add_argument("stack") + c.add_argument("suffix") + c.add_argument("--write-mbps", type=int, required=True) + c.add_argument("--write-iops", type=int, required=True) + d = sub.add_parser("compose") + d.add_argument("stack") + d.add_argument("args", nargs=argparse.REMAINDER) + for op in ("init", "up", "down"): + sub.add_parser(op).add_argument("stack") + k = sub.add_parser("check") + k.add_argument("host") + k.add_argument("--runner", action="store_true") + a = p.parse_args(argv) + try: + if a.op == "compose": + args = a.args[1:] if a.args[:1] == ["--"] else a.args + with contextlib.nullcontext() if websites.reads(args) else _alone(): + return compose(a.stack, args, capture=False) + if a.op == "configure": + path, settings = configure(a.stack, a.suffix, a.write_mbps, a.write_iops) + out = {"settings": path, **{k: v for k, v in settings.items() if k != TOKEN}} + elif a.op == "check": + if not HOST.fullmatch(a.host): + raise GitLabError(f"{a.host!r} is not GitLab's host") + out = {"url": ready(websites.SCHEME + a.host, _token(), runner=a.runner)} + else: + with _alone(): + out = {"init": init, "up": lambda s: up(s, say=print), "down": down}[a.op](a.stack) + except (GitLabError, websites.WebsiteError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/nullius/catalogue/benchmarks/osworld2/nop.py b/app/nullius/catalogue/benchmarks/osworld2/nop.py new file mode 100644 index 0000000..6393712 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/nop.py @@ -0,0 +1,91 @@ +"""OSWorld-V2's null control: the upstream's run_single_example on one task with an agent that does nothing. + + nop.py VM_IMAGE PASSWORD EXAMPLES DOMAIN TASK_ID RESULT_DIR + nop.py --instruction EXAMPLES DOMAIN TASK_ID OUT + +The agent answers DONE at its one step, and evaluate() on the untouched VM goes +to RESULT_DIR/result.txt. --instruction writes the task's instruction and +whether it asks for a proxy to OUT as JSON. A task with no class file is an +error. The setup gets SETUP_ATTEMPTS; a failed one goes to +RESULT_DIR/setup_failed.json by exception type only, since the message could +quote the gated task.""" +import argparse +import json +import logging +import os +import signal +import sys + +from task_loader import find_task_class_path, load_task_config, resolve_task_json_path + +SETUP_ATTEMPTS = 2 +SETUP_FAILED = "setup_failed.json" + + +class DoNothing: + def reset(self, *args, **kwargs): + pass + + def predict(self, instruction, obs): + return "", ["DONE"] + + +def _stop(signum, frame): + sys.exit(1) + + +def _fail(said): + logging.error(said) + sys.exit(1) + + +def _task(examples, domain, eid): + if not find_task_class_path(eid, examples, domain, "v2"): + _fail(f"no task class for {domain}__{eid} under {examples}") + try: + task = load_task_config(resolve_task_json_path(task_id=eid, base_dir=examples, domain=domain, eval_version="v2"), + task_id=eid, base_dir=examples, domain=domain, eval_version="v2") + instruction = task["instruction"] + except (Exception, SystemExit) as e: + _fail(f"the task class for {domain}__{eid} did not load: {type(e).__name__}" + + (", which may need a service this entry does not set up (the mock websites, GitLab)" + if isinstance(e, ValueError) else "")) + return task, instruction + + +logging.basicConfig(level=logging.INFO, format="[%(asctime)s %(levelname)s %(name)s] %(message)s") +if sys.argv[1:2] == ["--instruction"]: + examples, domain, eid, out = sys.argv[2:6] + task, instruction = _task(examples, domain, eid) + with open(out, "w", encoding="utf-8") as f: + json.dump({"instruction": instruction, "proxy": bool(task.get("proxy", False))}, f) + sys.exit(0) +import lib_run_single +from desktop_env import desktop_env +from desktop_env.desktop_env import DesktopEnv, EnvironmentSetupError + +vm, password, examples, domain, eid, out = sys.argv[1:7] +example, instruction = _task(examples, domain, eid) +signal.signal(signal.SIGTERM, _stop) +desktop_env.MAX_RETRIES = SETUP_ATTEMPTS +env = None +try: + env = DesktopEnv(path_to_vm=vm, action_space="pyautogui", provider_name="docker", screen_size=(1920, 1080), + headless=True, os_type="Ubuntu", require_a11y_tree=False, enable_proxy=True, + client_password=password, instance_type=example.get("instance_type"), + volume_size=example.get("volume_size"), force_disable_vnc=True, force_disable_recording=True) + lib_run_single.run_single_example(DoNothing(), env, example, 1, instruction, + argparse.Namespace(sleep_after_execution=3.0, result_dir=out, + checkpoint_eval_mode="off"), out, []) +except EnvironmentSetupError as e: + with open(os.path.join(out, SETUP_FAILED), "w", encoding="utf-8") as f: + json.dump({"error": type(e).__name__, "cause": type(e.__cause__).__name__ if e.__cause__ else None, + "attempts": SETUP_ATTEMPTS}, f) + logging.error(f"the null control's setup of {domain}__{eid} failed: {type(e).__name__}") + raise +except Exception as e: + logging.error(f"the null control's run of {domain}__{eid} failed: {type(e).__name__}") + raise +finally: + if env is not None: + env.close() diff --git a/app/nullius/catalogue/benchmarks/osworld2/proxy.py b/app/nullius/catalogue/benchmarks/osworld2/proxy.py new file mode 100644 index 0000000..ed79b16 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/proxy.py @@ -0,0 +1,372 @@ +"""OSWorld2's proxy for the tasks that ask for one: tinyproxy in a container of the unit's own. + + python3 proxy.py --build + python3 proxy.py --sweep UNIT_DIR RECORD_FD [CONTAINER] + python3 proxy.py --watch UNIT_DIR DRIVER_PID RECORD_FD CONTAINER + +The driver passes the unit's proxy.json as the upstream's PROXY_CONFIG_FILE. +The container is on NETWORK and publishes no port; tinyproxy logs warnings +only, never a request's URL. A container is labelled with the unit's folder +and named with this user's id and its driver's pid. A start docker did not +answer is never repeated: a sweeper (--sweep) removes the container later, +and a watcher (--watch) when the driver is killed first.""" +import contextlib +import fcntl +import hashlib +import json +import os +import re +import secrets +import socket +import subprocess +import sys +import time + +DOCKERFILE = r"""FROM alpine:3.24@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +RUN apk add --no-cache tinyproxy~1.11 +USER 65534:65534 +ENTRYPOINT ["/bin/sh", "-c", "printf '%s\\n' \"$TINYPROXY_CONF\" > /tmp/tinyproxy.conf && exec tinyproxy -d -c /tmp/tinyproxy.conf"] +""" +IMAGE = "nullius/osworld2-proxy:" + hashlib.sha256(DOCKERFILE.encode()).hexdigest()[:16] +CONF = "TINYPROXY_CONF" +PORT = 8888 +USER = "nullius" +FILE = "proxy.json" +NAME = "nullius-osworld2-proxy-" +UNIT = "nullius.osworld2.proxy" +FOLDER = "nullius.osworld2.proxy.dir" +PACKAGE = "nullius.osworld2.proxy.package" +NETWORK = "nullius-osworld2" +SWEEPS = (2, 8, 20, 90, 180, 300, 300) +RETRY_SECONDS = 120 +SWEEP_SECONDS = 1800 +CLIENTS = 100 +PIDS = 128 +WATCH_SECONDS = 1 +WRITE_BPS = 52428 +WRITE_IOPS = 1 +TAKEN = {"OSWORLD2_PROXY_WRITE_BPS": str(WRITE_BPS), "OSWORLD2_PROXY_WRITE_IOPS": str(WRITE_IOPS)} +LOG = "proxy.log" +DOCKER_SECONDS = 120 +START_SECONDS = 300 +_SEEN = {} +_STARTS = {} + + +class NoAnswer(RuntimeError): + pass + + +def _docker(*args, env=None, until=None): + seconds = DOCKER_SECONDS if until is None else min(DOCKER_SECONDS, round(until - time.monotonic(), 1)) + if seconds <= 0: + raise NoAnswer(f"docker {args[0]} was not asked: the time given for it ran out") + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, env=env) + except subprocess.TimeoutExpired: + raise NoAnswer(f"docker {args[0]} got no answer in {seconds:g} s: is its daemon running?") + except OSError: + raise RuntimeError("no docker on this machine's PATH: the proxy needs it") + + +def _last(said, otherwise): + """The last line docker said, past the pointer to its help it adds to every refusal.""" + lines = [ln.strip() for ln in (said or "").splitlines() if ln.strip() and not ln.startswith("Run 'docker")] + return (lines or [otherwise])[-1][:300] + + +def _mark(dir): + return hashlib.sha256(os.path.abspath(dir).encode()).hexdigest()[:32] + + +def _prefix(): + return f"{NAME}{os.getuid()}-" + + +def config(token): + """No Via header, since the pool's residential proxies add none.""" + return "".join(f"{line}\n" for line in (f"Port {PORT}", "Timeout 600", f"MaxClients {CLIENTS}", "ConnectPort 443", + "ConnectPort 80", "LogLevel Warning", "DisableViaHeader Yes", + f"BasicAuth {USER} {token}")) + + +def _alive(pid): + """Whether the process runs; one that has exited and waits for its parent to see it does not.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + try: + with open(f"/proc/{pid}/stat", encoding="utf-8", errors="replace") as f: + return f.read().rpartition(")")[2].split()[0] not in ("Z", "X") + except (OSError, IndexError): + return True + + +def _orphans(until=None): + """Remove this user's proxies whose driver is gone or whose unit holds no record; another user's stay.""" + seen = _docker("ps", "-a", "--filter", f"name={_prefix()}", "--format", "{{.Names}}\t{{.Label \"" + FOLDER + "\"}}", + until=until) + gone = [] + for name, _, folder in (ln.partition("\t") for ln in seen.stdout.splitlines()): + m = re.fullmatch(re.escape(_prefix()) + r"(\d+)-\d+", name.strip()) + if m and (not _alive(int(m.group(1))) or (folder and not os.path.lexists(os.path.join(folder, FILE)))): + gone.append(name.strip()) + if gone: + _docker("rm", "-f", *gone, until=until) + + +def _image(dir, until): + """Orphans removed and the image checked at a process's first start; its ID logged at each.""" + if not _SEEN: + _orphans(until) + seen = _docker("image", "inspect", "--format", "{{.Id}}\t{{json .Config}}", IMAGE, until=until) + if seen.returncode != 0: + raise RuntimeError(f"no {IMAGE} on this machine, the proxy of the tasks that ask for one: build it " + "once with python3 proxy.py --build, from this benchmark's folder (it needs the " + "network)") + ident, _, config = seen.stdout.strip().partition("\t") + try: + package = ((json.loads(config) or {}).get("Labels") or {}).get(PACKAGE, "") + except (ValueError, AttributeError): + package = "" + _SEEN.update(id=ident or IMAGE, package=package) + _note(dir, f"proxy image {_SEEN['id']}, " + (_SEEN["package"] or "its tinyproxy package not labelled: " + "build it again with python3 proxy.py --build")) + + +def _ready(name, until): + """A stopped container is an error with its exit code, never its log, which may name a request's host.""" + host = "" + late = f"did not answer in its start's {START_SECONDS} s" + while True: + try: + seen = _docker("inspect", "-f", "{{.State.Status}}\t{{.State.ExitCode}}\t{{with index .NetworkSettings." + "Networks " + json.dumps(NETWORK) + "}}{{.IPAddress}}{{end}}", name, until=until) + except NoAnswer: + if time.monotonic() < until: + raise + raise RuntimeError(f"the proxy at {host or 'no address'}:{PORT} {late}") from None + status, code, host = (seen.stdout.strip().split("\t") + ["", ""])[:3] + if status not in ("running", "created"): + raise RuntimeError(f"the proxy's container stopped, exit code {code or 'unknown'}") + if status == "running" and host: + try: + socket.create_connection((host, PORT), timeout=1).close() + return host + except OSError: + pass + if time.monotonic() >= until: + raise RuntimeError(f"the proxy at {host or 'no address'}:{PORT} {late}") + time.sleep(0.2) + + +def start(dir, disk=None): + """The unit's proxy started, and its pool file's path; a proxy left from an earlier attempt is removed first.""" + until = time.monotonic() + START_SECONDS + _image(dir, until) + remove(dir, final=True, until=until) + path = os.path.join(dir, FILE) + name = f"{_prefix()}{os.getpid()}-{time.time_ns()}" + with _locked(dir): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + _STARTS[os.path.abspath(dir)] = name + _watcher(dir, fd, name) + finally: + os.close(fd) + token = secrets.token_hex(16) + limits = [] if disk is None else ["--device-write-bps", f"{disk}:{WRITE_BPS}", "--device-write-iops", f"{disk}:{WRITE_IOPS}"] + try: + made = _docker("run", "-d", "--name", name, "--label", f"{UNIT}={_mark(dir)}", + "--label", f"{FOLDER}={os.path.abspath(dir)}", "--network", NETWORK, + "--read-only", "--tmpfs", "/tmp:rw,noexec,nosuid,size=1m,mode=1777", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--memory", "64m", "--pids-limit", str(PIDS), + "--cpus", "1", "--log-driver", "local", "--log-opt", "max-size=1m", "--log-opt", "max-file=1", + "--log-opt", "compress=false", *limits, + "-e", CONF, IMAGE, env=dict(os.environ, **{CONF: config(token)}), until=until) + except NoAnswer as e: + _found(dir, name, e, until) + made = None + _record(path, []) + if made is not None and made.returncode != 0: + raise RuntimeError("the proxy's container was refused: " + _last(made.stderr, "no reason")) + host = _ready(name, until) + _record(path, [{"host": host, "port": PORT, "username": USER, "password": token, "protocol": "http"}]) + return path + + +def _found(dir, name, unanswered, until): + """No container from the unanswered start, or no answer, is an error; the record stays empty for its sweeper.""" + later = "; the unit's sweeper removes one it makes later" + try: + found = _docker("ps", "-a", "-q", "--filter", f"label={UNIT}={_mark(dir)}", "--filter", f"name={name}", + until=until) + except NoAnswer as e: + raise RuntimeError(f"{unanswered}, and then {e}: whether it made the proxy's container is unknown{later}") + if found.returncode != 0: + raise RuntimeError(f"{unanswered}, and could not say whether it made the proxy's container: " + + _last(found.stderr, "no reason") + later) + if not found.stdout.split(): + raise RuntimeError(f"{unanswered}, and has made no container for the proxy yet{later}") + + +@contextlib.contextmanager +def _locked(dir): + fd = os.open(dir, os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) + + +def _ours(dir, fd): + """Whether the unit's record is still the file ``fd`` holds open, so no later start has replaced it.""" + try: + here, own = os.lstat(os.path.join(dir, FILE)), os.fstat(fd) + except OSError: + return False + return (here.st_dev, here.st_ino) == (own.st_dev, own.st_ino) + + +def _record(path, pool): + with os.fdopen(os.open(path, os.O_WRONLY | os.O_TRUNC | os.O_NOFOLLOW), "w", encoding="utf-8") as f: + json.dump(pool, f) + + +def remove(dir, final=False, until=None, name=None, fd=None): + """Remove the unit's proxy, then its record; an empty record stays until the ``final`` sweep.""" + path = os.path.join(dir, FILE) + if not (os.path.lexists(path) if fd is None else _ours(dir, fd)): + return + found = _docker("ps", "-a", "-q", "--filter", f"label={UNIT}={_mark(dir)}", "--filter", f"name={name or _prefix()}", + until=until) + if found.returncode != 0: + raise RuntimeError("the unit's proxy was not looked for: " + _last(found.stderr, "no reason")) + ids = found.stdout.split() + if ids: + gone = _docker("rm", "-f", *ids, until=until) + if gone.returncode != 0: + raise RuntimeError("the unit's proxy was not removed: " + _last(gone.stderr, "no reason")) + if fd is None: + with contextlib.suppress(FileNotFoundError): + if final or os.lstat(path).st_size: + os.remove(path) + return + with _locked(dir): + if _ours(dir, fd) and (final or os.fstat(fd).st_size): + os.remove(path) + + +def _removed(dir, final, name, fd, until): + try: + remove(dir, final, until=until, name=name, fd=fd) + except (RuntimeError, OSError) as e: + return str(e) or type(e).__name__ + return None + + +def sweep(dir, fd, name=None): + """Remove the start's container after each of SWEEPS; a removal that never succeeds is a line in LOG.""" + until = time.monotonic() + SWEEP_SECONDS + failed = None + for i, wait in enumerate(SWEEPS): + if not _ours(dir, fd): + break + left = until - time.monotonic() - 2 * DOCKER_SECONDS + final = i == len(SWEEPS) - 1 or left < wait + time.sleep(max(0, min(wait, left))) + failed = _removed(dir, final, name, fd, until) + if final: + break + while failed and _ours(dir, fd) and until - time.monotonic() > RETRY_SECONDS: + time.sleep(RETRY_SECONDS) + failed = _removed(dir, True, name, fd, until) + if failed: + _note(dir, f"the unit's proxy was not removed in {SWEEP_SECONDS // 60} min: {failed}") + with contextlib.suppress(RuntimeError, OSError): + _orphans(until) + + +def watch(dir, pid, fd, name): + """Sweep the start's container once its driver is gone, while the record is still the start's.""" + while _ours(dir, fd): + if not _alive(pid): + sweep(dir, fd, name) + return + time.sleep(WATCH_SECONDS) + + +def _spawn(fd, *args): + subprocess.Popen([sys.executable, os.path.abspath(__file__), *args], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, pass_fds=(fd,)) + + +def _later(dir, name=None): + with contextlib.suppress(OSError): + fd = os.open(os.path.join(dir, FILE), os.O_RDONLY | os.O_NOFOLLOW) + try: + _spawn(fd, "--sweep", os.path.abspath(dir), str(fd), *([name] if name else [])) + finally: + os.close(fd) + + +def _watcher(dir, fd, name): + _spawn(fd, "--watch", os.path.abspath(dir), str(os.getpid()), str(fd), name) + + +def _note(dir, line): + with contextlib.suppress(OSError): + fd = os.open(os.path.join(dir, LOG), os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o644) + with os.fdopen(fd, "a", encoding="utf-8") as f: + f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + line + "\n") + + +@contextlib.contextmanager +def held(dir, wanted, disk=lambda: None): + """The unit's variables while its proxy runs, when ``wanted``; a record left after the removal gets a sweeper.""" + if not wanted: + yield {} + return + try: + device = disk() + yield {"PROXY_CONFIG_FILE": start(dir, device), **({} if device is None else TAKEN)} + finally: + try: + remove(dir) + except (RuntimeError, OSError) as e: + _note(dir, f"the unit's proxy was not removed: {e}") + if os.path.lexists(os.path.join(dir, FILE)): + _later(dir, _STARTS.get(os.path.abspath(dir))) + + +def build(): + """Build the image, then again from the cache, labelled with its tinyproxy package.""" + if subprocess.run(["docker", "build", "-t", IMAGE, "-"], input=DOCKERFILE, text=True).returncode != 0: + return 1 + seen = subprocess.run(["docker", "run", "--rm", "--network", "none", "--entrypoint", "apk", IMAGE, "list", + "--installed", "tinyproxy"], capture_output=True, text=True) + package = (seen.stdout.split() or [""])[0] if seen.returncode == 0 else "" + if not package.startswith("tinyproxy-"): + print("the image's tinyproxy package was not found: " + _last(seen.stderr, "apk said nothing"), file=sys.stderr) + return 1 + if subprocess.run(["docker", "build", "--label", f"{PACKAGE}={package}", "-t", IMAGE, "-"], input=DOCKERFILE, + text=True).returncode != 0: + return 1 + print(IMAGE, package) + return 0 + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--sweep"] and len(sys.argv) in (4, 5): + sweep(sys.argv[2], int(sys.argv[3]), *sys.argv[4:]) + sys.exit(0) + if sys.argv[1:2] == ["--watch"] and len(sys.argv) == 6: + watch(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5]) + sys.exit(0) + if sys.argv[1:] != ["--build"]: + sys.exit("\n".join(ln.strip() for ln in __doc__.splitlines()[2:5])) + sys.exit(build()) diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json new file mode 100644 index 0000000..5601d44 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -0,0 +1,382 @@ +{ + "schema": "nullius.benchmark/v1", + "key": "osworld2", + "label": "OSWorld2", + "summary": "Each task is a unit: a long, multi-app task on a real Ubuntu desktop in its own VM, set up by the task's own class, one of OSWorld-V2's own agents (M3 or Muse Spark) acting on it from screenshots, and the task's own evaluate() scoring the VM's final state from 0 to 1.", + "family": "agent", + "gated": true, + "needs": [ + "agent.computer/v1" + ], + "upstream": { + "repository": "https://github.com/xlang-ai/OSWorld-V2", + "tag": "osworld-v2.1", + "tested": "3d778a3c9a34a079316f70df023b166700445792", + "tested_date": "2026-09-16", + "license": "Apache-2.0" + }, + "driver": { + "command": [ + "{install}/.venv/bin/python", + "{here}/driver.py", + "{here}/agents.json", + "{here}/nop.py", + "{here}/startup/sitecustomize.py", + "{here}/websites.compose.yml", + "{here}/gitlab.compose.yml" + ], + "install": { + "python": "3.12", + "project": ".", + "locked": true + } + }, + "datasets": [ + { + "id": "offline36", + "label": "Offline, no services", + "summary": "The 36 tasks of the offline list that need no service: the VM, the gated assets, and PyPI for the Python packages some of them install in the VM.", + "units": 36 + }, + { + "id": "no_services63", + "label": "No hosted services", + "summary": "The 63 tasks that need no service hosted for them: Offline, no services and 27 that need the LLM grader, the simulated user or the public internet. No mock website, GitLab or proxy.", + "units": 63 + }, + { + "id": "websites35", + "label": "Mock website tasks", + "summary": "The 35 tasks that need the mock websites and no GitLab or proxy: OSWorld-web served on this machine, its suffix given as Website host suffix. 2 of them use Overleaf and run one at a time.", + "units": 35 + }, + { + "id": "websites98", + "label": "No GitLab or proxy", + "summary": "No hosted services and Mock website tasks: the 98 tasks that need no GitLab or proxy.", + "units": 98 + }, + { + "id": "test_v2_offline_no_internet", + "label": "Offline", + "summary": "The upstream's test_v2_offline_no_internet.json; 1 of its tasks needs GitLab, 3 a proxy, and 33 more the mock websites.", + "path": "evaluation_examples/test_v2_offline_no_internet.json", + "sha256": "a14975fe0972530993d634d380ba574ecbd19b730d14716a392efe02faa62257", + "units": 82 + }, + { + "id": "test_v2", + "label": "All tasks", + "summary": "The upstream's test_v2.json, its runners' default; 2 of its tasks need GitLab, 8 a proxy, and 35 more the mock websites.", + "path": "evaluation_examples/test_v2.json", + "sha256": "7f2c0f531d035dd6f5886c492b96739534385f8e5369e4c764295ad52bf6e7f4", + "units": 108 + } + ], + "units": { + "noun": "task", + "nouns": "tasks" + }, + "score": { + "kind": "score", + "says": "OSWorld-V2's own scoring: the task class's evaluate() on the VM's final state, as run_single_example writes it to result.txt (0 to 1, partial credit; result.json carries the detail and safety checks); the upstream's score is the mean", + "judge": { + "model": "anthropic/claude-sonnet-4.6", + "says": "the upstream's LLM grader in the evaluate() of 21 tasks, Claude Sonnet 4.6 over Anthropic's Messages API as run_multienv_m3.sh sets it (OSWORLD_EVAL_MODEL_*), whichever agent runs", + "pin": { + "openrouter": [ + "anthropic" + ] + } + } + }, + "limits": { + "score_seconds": 3600, + "call_seconds": 3300, + "run_again": false + }, + "setups": { + "original": { + "summary": "OSWorld-V2's own agent as published, M3 or Muse Spark (the agent option): its script on the task, with its own prompts, parser and history, asking its model through the relay, MiniMax-M3 over Anthropic's Messages API or muse-spark-1.3 over OpenAI's Responses API, on the docker provider's VM, scored by the task's evaluate().", + "reference_agent": "OSWorld-V2 agents", + "models": [ + "MiniMax-M3", + "muse-spark-1.3" + ], + "default": "MiniMax-M3", + "side_models": [ + "claude-sonnet-4-6" + ], + "served_as": { + "MiniMax-M3": "minimax/minimax-m3", + "muse-spark-1.3": "meta/muse-spark-1.3", + "claude-sonnet-4-6": "anthropic/claude-sonnet-4.6" + }, + "providers": { + "openrouter": "", + "custom": "" + }, + "pin": { + "openrouter": { + "MiniMax-M3": [ + "minimax" + ], + "muse-spark-1.3": [ + "meta" + ], + "claude-sonnet-4-6": [ + "anthropic" + ] + } + }, + "turn_seconds": 86400, + "paper": "the OSWorld-V2 paper, Table 3 (Section 3.2), MiniMax M3", + "comparable": [ + "MiniMax-M3" + ] + } + }, + "options": { + "agent": { + "label": "Agent", + "summary": "Which of OSWorld-V2's own agents runs: each its script as published, with its own prompts, tools and history, asking the model it is published with.", + "values": [ + "m3", + "muse_spark" + ], + "default": "m3", + "setups": [ + "original" + ], + "about": { + "m3": "MiniMax's M3Agent: run_multienv_m3.py, MiniMax-M3 over Anthropic's Messages API", + "muse_spark": "Meta's Muse Spark: run_multienv_muse_spark.py, muse-spark-1.3 over the Responses API" + } + }, + "max_steps": { + "label": "Steps", + "summary": "--max_steps: 100 as run_multienv_m3.sh invokes M3, 300 for a cheaper run, or 500, M3's budget in the paper and Muse Spark's in its wrapper and published run.", + "values": [ + "100", + "300", + "500" + ], + "default": "100", + "setups": [ + "original" + ], + "about": { + "100": "run_multienv_m3.sh's limit", + "300": "below the paper's budget, for a cheaper run", + "500": "the paper's budget for M3, and Muse Spark's" + } + }, + "thinking": { + "summary": "M3_THINKING_MODE: off, as run_multienv_m3.sh leaves it unset, or adaptive, as the paper ran M3 (thinking on).", + "values": [ + "off", + "adaptive" + ], + "default": "off", + "setups": [ + "original" + ], + "about": { + "off": "no thinking field, temperature 0.6", + "adaptive": "thinking={'type': 'adaptive'}, temperature 1" + }, + "when": { + "agent": [ + "m3" + ] + } + }, + "vm_provider": { + "label": "VM provider", + "summary": "--provider_name: docker only; aws, azure, vmware and virtualbox need accounts or a hypervisor of their own.", + "values": [ + "docker" + ], + "default": "docker", + "setups": [ + "original", + "yours" + ], + "about": { + "docker": "happysixd/osworld-docker (pinned by digest) running the v2 Ubuntu qcow2 under QEMU, KVM when /dev/kvm is there" + }, + "needs": { + "docker": { + "host": [ + "docker", + "scoring_programs" + ] + } + } + }, + "websites": { + "label": "Website host suffix", + "summary": "WEBSITE_HOST_SUFFIX: none, or the host suffix OSWorld-web's sites are served under for this machine's VMs, a wildcard DNS name of the address the stack listens on, the gateway of the VMs' own network (10.213.0.1.nip.io). The 40 tasks that use the sites need it.", + "pattern": "none|(?=.{3,200}$)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "gitlab": { + "label": "GitLab host", + "summary": "GITLAB_URL: none, or the host name of Task-Web/gitlab behind the mock websites' Caddy (gitlab.10.213.0.1.nip.io), reached over http. Each unit of tasks 026 and 041 starts it from its snapshot and removes it after; they need its root token, the licence GITLAB_PRIVATE_TOKEN.", + "pattern": "none|[a-z0-9](?:[a-z0-9.-]{0,56}[a-z0-9])?(?::[1-9][0-9]{0,4})?", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "gitlab_write": { + "label": "GitLab write share", + "summary": "Docker's write limits of a GitLab unit's own GitLab, on the disk under Docker's root: none, or MB/s:IOPS for all of its containers together. With the VMs' writes limited, tasks 026 and 041 need it, a slot of the machine's budget beside their VM's.", + "pattern": "none|[1-9][0-9]{0,5}:[1-9][0-9]{0,5}", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "write_mbps": { + "label": "Disk write MB/s", + "summary": "Docker's device_write_bps on each VM's container, on the disk under Docker's root: none, or MB/s (1024 * 1024 bytes). Each container has its own: with -j N, give each the machine's budget divided by N.", + "pattern": "none|[1-9][0-9]{0,5}", + "default": "none", + "setups": [ + "original", + "yours" + ] + }, + "write_iops": { + "label": "Disk write IOPS", + "summary": "Docker's device_write_iops on each VM's container, on the disk under Docker's root: none, or write operations a second, each container its own.", + "pattern": "none|[1-9][0-9]{0,5}", + "default": "none", + "setups": [ + "original", + "yours" + ] + } + }, + "host": { + "docker": { + "summary": "Docker on the machine the driver runs on, usable without sudo, for the upstream's docker provider; /dev/kvm makes the VMs fast, and the upstream runs them without it, slowly; the ip_tables and iptable_nat kernel modules loaded, or the VM's port forwarding fails; the subnet 10.213.0.0/24 free, for the VMs' network nullius-osworld2 that Nullius makes.", + "check": [ + "docker" + ] + }, + "scoring_programs": { + "summary": "ImageMagick, ffmpeg, ffprobe and tesseract on the driver's machine, where the evaluate() of tasks 025, 046, 051, 056, 085 and 095 runs them.", + "check": [ + "identify", + "convert", + "ffmpeg", + "ffprobe", + "tesseract" + ], + "install": { + "apt": [ + "imagemagick", + "ffmpeg", + "tesseract-ocr" + ] + } + } + }, + "environments": { + "gitlab": { + "summary": "GitLab's root token for the driver, which tasks 026 and 041 use to set up and score: the base image, nothing added.", + "dockerfile": "gitlab.Dockerfile", + "local_ok": true, + "needs_secrets": [ + { + "name": "GITLAB_PRIVATE_TOKEN", + "to": "driver", + "summary": "GitLab's root personal access token, the one python3 gitlab_server.py init made it with" + } + ] + } + }, + "controls": { + "nop": { + "source": "nullius", + "from": "Nullius: the upstream's run_single_example with an agent whose one action is DONE, then the task's evaluate() on the untouched VM.", + "expect": { + "at_most": 0.01 + } + } + }, + "proven": { + "offline36": { + "units": { + "units": 36, + "passed": "36 ids, the same twice, unique", + "on": "2026-11-04" + }, + "nop": { + "passed": "0/4", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 4, + "job": "2026-11-04__23-15-30__osworld2__offline36__prove-nop", + "source": "nullius", + "on": "2026-11-04", + "limit": 4 + } + }, + "test_v2": { + "units": { + "units": 108, + "passed": "108 ids, the same twice, unique", + "on": "2026-11-15" + }, + "nop": { + "passed": "0/108", + "value": 0.0021, + "expect": { + "at_most": 0.01 + }, + "units": 108, + "job": "2026-11-15__06-18-44__osworld2__test_v2__prove-nop", + "source": "nullius", + "options": [ + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "gitlab.10.213.0.1.nip.io", + "gitlab_write": "16:160", + "write_mbps": "16", + "write_iops": "160" + }, + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "none", + "gitlab_write": "none", + "write_mbps": "16", + "write_iops": "160" + }, + { + "vm_provider": "docker", + "websites": "10.213.0.1.nip.io", + "gitlab": "none", + "gitlab_write": "none", + "write_mbps": "24", + "write_iops": "240" + } + ], + "judge": "anthropic/claude-sonnet-4.6", + "on": "2026-11-15" + } + } + } +} diff --git a/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py b/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py new file mode 100644 index 0000000..0b01f8e --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/startup/sitecustomize.py @@ -0,0 +1,457 @@ +"""OSWorld2's VM container create and guest disk, in every Python process the driver starts. + +The driver puts this folder first on PYTHONPATH. For the provider's image only, +it wraps docker-py's ContainerCollection.create: CREATE_SECONDS a call, a +create that timed out on reading looked for by its label instead of sent +again, the container on NETWORK (on the default bridge a Docker in the VM +takes the host's address), the write limits on the disk under Docker's root +and its ports, the guest's PORTS too, on 127.0.0.1. A docker DesktopEnv gets a disk of at +least GUEST_GB, as the upstream's aws provider gives. With another vm_provider +it wraps nothing. Once OSWORLD2_STOP's file exists, an agent's step ends with DONE. +A log record never shows the password of PROXY_CONFIG_FILE's proxy.""" +import contextlib +import functools +import inspect +import json +import logging +import os +import re +import signal +import subprocess +import sys +import time +import uuid + +PROVIDER = "happysixd/osworld-docker" +CREATE_SECONDS = 120 +ADOPT_SECONDS = 60 +LOOK_SECONDS = 10 +POLL_SECONDS = 2 +SWEEP_SECONDS = 1800 +LABEL = "nullius.osworld2.create" +NETWORK = "nullius-osworld2" +SUBNET = "10.213.0.0/24" +GATEWAY = "10.213.0.1" +NETWORK_LABEL = "nullius.osworld2.network" +PORTS = "OSWORLD2_GUEST_PORTS" +LOOPBACK = "127.0.0.1" +DOCKER_PY_SECONDS = 60 +PORT_LISTS = 4 +LONGEST_START = (PORT_LISTS * DOCKER_PY_SECONDS + 5 * CREATE_SECONDS + + ADOPT_SECONDS + POLL_SECONDS + 2 * LOOK_SECONDS) +LIMITS = (("device_write_bps", "NULLIUS_OPTION_WRITE_MBPS", 1024 * 1024, "BlkioDeviceWriteBps", "--device-write-bps"), + ("device_write_iops", "NULLIUS_OPTION_WRITE_IOPS", 1, "BlkioDeviceWriteIOps", "--device-write-iops")) +TAKEN = {"device_write_bps": "OSWORLD2_PROXY_WRITE_BPS", "device_write_iops": "OSWORLD2_PROXY_WRITE_IOPS"} +SYS_BLOCK = "/sys/dev/block" +DEV = "/dev" +ACTION = "run where Docker limits writes on the disk under its root, or set Disk write MB/s and Disk write IOPS to none" +_DISK = {} +GUEST_GB = 40 +DESKTOP_ENV = "desktop_env.desktop_env" +EPISODE = "lib_run_single" +DOCKER_PROVIDER = "desktop_env.providers.docker.provider" +STOP = "OSWORLD2_STOP" +MARK = "_nullius_osworld2" +HIDDEN = "" + + +class WriteLimitError(RuntimeError): + pass + + +class NetworkError(RuntimeError): + pass + + +def network(docker): + """Made when missing, a concurrent maker no error; one of that name on another subnet or gateway is a NetworkError.""" + shape = "{{index .Labels \"" + NETWORK_LABEL + "\"}}|{{range .IPAM.Config}}{{.Subnet}} {{.Gateway}} {{end}}" + seen = docker("network", "inspect", "-f", shape, NETWORK) + if seen.returncode != 0: + made = docker("network", "create", "--driver", "bridge", "--subnet", SUBNET, "--gateway", GATEWAY, + "--label", f"{NETWORK_LABEL}=1", NETWORK) + seen = docker("network", "inspect", "-f", shape, NETWORK) + if seen.returncode != 0: + said = ((made.stderr or "").strip().splitlines() or ["no reason"])[-1][:300] + raise NetworkError(f"the VMs' network {NETWORK} ({SUBNET}) could not be made: {said}; free that " + "subnet on this machine, or make the network with that subnet and gateway " + f"{GATEWAY}") + label, _, found = seen.stdout.strip().partition("|") + if found.split() != [SUBNET, GATEWAY]: + have = f"the network {NETWORK} has {found.strip() or 'no subnet'}, not {SUBNET} with gateway {GATEWAY}" + if label: + raise NetworkError(f"{have}: remove it (docker network rm {NETWORK}) once no container uses it") + raise NetworkError(f"{have}, and Nullius did not make it: leave it to its owner, who can give it another " + f"name, or run on a machine where the name {NETWORK} and {SUBNET} are free") + return NETWORK + + +def wanted(environ): + """The options' limits as (argument, rate, HostConfig field, CLI flag), less the proxy's share.""" + out = [] + for key, name, unit, field, flag in LIMITS: + value = environ.get(name, "none") + if value != "none": + if not re.fullmatch(r"[1-9][0-9]*", value): + raise WriteLimitError(f"the VM's disk write limit cannot be applied: {name} is {value!r}; {ACTION}") + taken = environ.get(TAKEN[key], "0") + if not re.fullmatch(r"0|[1-9][0-9]*", taken) or int(value) * unit <= int(taken): + raise WriteLimitError(f"the VM's disk write limit cannot be applied: {name} is {value}, and the " + f"unit's proxy takes {taken} of it ({TAKEN[key]}); give the VM more") + out.append((key, int(value) * unit - int(taken), field, flag)) + return out + + +def published(environ): + """The image forwards every TCP port but its web page's to the guest, so the same number reaches it.""" + return {int(p): (LOOPBACK, int(p)) for p in environ.get(PORTS, "").split()} + + +def disk(path): + """The block device under ``path``, a partition's parent disk; None when there is no block device.""" + st = os.stat(path).st_dev + node = os.path.join(SYS_BLOCK, f"{os.major(st)}:{os.minor(st)}") + if not os.path.isdir(node): + return None + node = os.path.realpath(node) + if os.path.exists(os.path.join(node, "partition")): + node = os.path.dirname(node) + try: + with open(os.path.join(node, "uevent"), encoding="utf-8") as f: + name = dict(ln.strip().partition("=")[::2] for ln in f).get("DEVNAME") + except OSError: + return None + return os.path.join(DEV, name) if name and os.path.exists(os.path.join(DEV, name)) else None + + +def root_disk(root): + """The disk under Docker's root ``root``, found once in a process; no disk is a WriteLimitError.""" + if root not in _DISK: + try: + device = disk(root) + except OSError as e: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker's root {root}: " + f"{type(e).__name__}; {ACTION}") from e + if device is None: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker's root {root} is on no " + f"block device {SYS_BLOCK} names; {ACTION}") + _DISK[root] = device + return _DISK[root] + + +def _throttle(client, limits, kwargs): + if not _DISK: + try: + root = client.info()["DockerRootDir"] + except Exception as e: + raise WriteLimitError(f"the VM's disk write limit cannot be applied: no Docker root from the daemon, " + f"{type(e).__name__}; {ACTION}") from e + root_disk(root) + device = next(iter(_DISK.values())) + return {key: list(kwargs.get(key) or []) + [{"Path": device, "Rate": rate}] for key, rate, _, _ in limits} + + +def _confirm(container, limits, given): + """A daemon that cannot limit writes drops them silently, so a container without them is removed before it starts.""" + try: + container.reload() + host = container.attrs.get("HostConfig") or {} + kept = all(given[key][-1] in (host.get(field) or []) for key, _, field, _ in limits) + except Exception: + kept = False + if not kept: + try: + container.remove(force=True, v=True) + except Exception: + pass + raise WriteLimitError(f"the VM's disk write limit cannot be applied: Docker created the VM's container " + f"without it; {ACTION}") + + +def _patient(client): + api = getattr(client, "api", None) + if api is not None and isinstance(getattr(api, "timeout", None), (int, float)) and api.timeout < CREATE_SECONDS: + api.timeout = CREATE_SECONDS + + +def _gave_up(error): + """Only a read timeout leaves the create undecided; any other error stands.""" + try: + from requests import exceptions + except ImportError: + return False + return isinstance(error, exceptions.ReadTimeout) + + +def _labelled(labels, token): + if isinstance(labels, dict): + out = dict(labels) + else: + out = dict.fromkeys(labels or [], "") + out[LABEL] = token + return out + + +def _made_late(collection, token): + """Each look waits LOOK_SECONDS at most, so the search ends within ADOPT_SECONDS, a pause and one look.""" + api = getattr(collection.client, "api", None) + kept = getattr(api, "timeout", None) + if api is not None and kept is not None: + api.timeout = LOOK_SECONDS + t0 = time.monotonic() + try: + while True: + try: + found = collection.list(all=True, filters={"label": f"{LABEL}={token}"}) + except Exception: + found = [] + if found: + return found[0] + if time.monotonic() - t0 >= ADOPT_SECONDS: + return None + time.sleep(POLL_SECONDS) + finally: + if api is not None and kept is not None: + api.timeout = kept + + +def _sweeper(token): + """A sweeper for the create's container, called off by _kept, and set going when this process closes its pipe or dies.""" + r, w = os.pipe() + try: + subprocess.Popen([sys.executable, "-I", os.path.abspath(__file__), token], stdin=r, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) + finally: + os.close(r) + return w + + +def _kept(fd): + try: + os.write(fd, b"kept") + except OSError: + pass + finally: + os.close(fd) + + +def sweep(token): + """The container of an abandoned create, should Docker make it within SWEEP_SECONDS, removed with its volume.""" + until = time.monotonic() + SWEEP_SECONDS + while time.monotonic() < until: + try: + found = subprocess.run(["docker", "ps", "-a", "-q", "--filter", f"label={LABEL}={token}"], + capture_output=True, text=True, timeout=CREATE_SECONDS).stdout.split() + if found and subprocess.run(["docker", "rm", "-f", "-v", *found], capture_output=True, + timeout=CREATE_SECONDS).returncode == 0: + return + except (OSError, subprocess.SubprocessError): + pass + time.sleep(LOOK_SECONDS) + + +def wrap_docker_create(environ=os.environ): + """Once a process: a create already wrapped, by an earlier run of this module, is left as it is.""" + limits, ports = wanted(environ), published(environ) + try: + from docker.models.containers import ContainerCollection + except ImportError: + return + create = ContainerCollection.create + if getattr(create, MARK, False): + return + + @functools.wraps(create) + def wrapped(self, image, *args, **kwargs): + if not (isinstance(image, str) and image.split("@")[0].split(":")[0] == PROVIDER): + return create(self, image, *args, **kwargs) + _patient(self.client) + given = _throttle(self.client, limits, kwargs) if limits else {} + token = uuid.uuid4().hex + swept = _sweeper(token) + placed = {} if kwargs.get("network") or kwargs.get("network_mode") else {"network": NETWORK} + if ports or kwargs.get("ports"): + placed["ports"] = {k: v if isinstance(v, tuple) else (LOOPBACK, v) + for k, v in {**(kwargs.get("ports") or {}), **ports}.items()} + try: + container = create(self, image, *args, **dict(kwargs, labels=_labelled(kwargs.get("labels"), token), + **placed, **given)) + except Exception as e: + if not _gave_up(e): + _kept(swept) + raise + container = _made_late(self, token) + if container is None: + os.close(swept) + raise + logging.getLogger("nullius.osworld2").warning( + "Docker made the VM's container after its client gave up (%s); that one is started, none is " + "created again", type(e).__name__) + _kept(swept) + if limits: + _confirm(container, limits, given) + container.remove = functools.partial(container.remove, v=True) + return container + + setattr(wrapped, MARK, True) + ContainerCollection.create = wrapped + + +def guest_gb(provider_name, volume_size): + """The task's own volume_size, or GUEST_GB when that is less; another provider's is left alone.""" + if str(provider_name).lower().strip() != "docker": + return volume_size + return max(int(volume_size or 0), GUEST_GB) + + +def floor_desktop_env(module): + """DesktopEnv's __init__ given guest_gb's volume_size, however its caller passes it.""" + cls = getattr(module, "DesktopEnv", None) + if cls is None or getattr(cls.__init__, MARK, False): + return + init = cls.__init__ + signature = inspect.signature(init) + if not {"provider_name", "volume_size"} <= set(signature.parameters): + return + default = signature.parameters["provider_name"].default + + @functools.wraps(init) + def wrapped(self, *args, **kwargs): + bound = signature.bind(self, *args, **kwargs) + given = bound.arguments + given["volume_size"] = guest_gb(given.get("provider_name", default), given.get("volume_size")) + return init(*bound.args, **bound.kwargs) + + setattr(wrapped, MARK, True) + cls.__init__ = wrapped + + +def queue_starts(module): + """Every VM start waits its turn for the provider's port lock, a restart as the first start.""" + module.LOCK_TIMEOUT = -1 + + +def _took(path): + """The stop's reasons as it ended a step, in ``path``.taken: one written once the episode ended is not its own.""" + with contextlib.suppress(OSError), open(path, encoding="utf-8") as f, open(path + ".taken", "x") as out: + out.write(f.read()) + + +def _stopping(predict, path): + @functools.wraps(predict) + def wrapped(*args, **kwargs): + if os.path.exists(path): + _took(path) + return "", ["DONE"] + try: + out = predict(*args, **kwargs) + except Exception: + if os.path.exists(path): + _took(path) + return "", ["DONE"] + raise + if not os.path.exists(path): + return out + _took(path) + return out[0], list(out[1]) + ["DONE"] + + setattr(wrapped, MARK, True) + return wrapped + + +def stop_when_asked(module): + run, path = getattr(module, "run_single_example", None), os.environ.get(STOP) + if run is None or not path or getattr(run, MARK, False): + return + + @functools.wraps(run) + def wrapped(agent, *args, **kwargs): + if not getattr(agent.predict, MARK, False): + agent.predict = _stopping(agent.predict, path) + return run(agent, *args, **kwargs) + + setattr(wrapped, MARK, True) + module.run_single_example = wrapped + + +class _Loader: + """The finder's loader, unchanged, with the module's PATCHES after its exec_module; the rest is the loader's.""" + + def __init__(self, loader): + self.loader = loader + + def __getattr__(self, name): + return getattr(self.loader, name) + + def create_module(self, spec): + return self.loader.create_module(spec) + + def exec_module(self, module): + self.loader.exec_module(module) + PATCHES[module.__name__](module) + + +class _Floor: + """Gives a PATCHES module's spec a _Loader on each import and reload; known by MARK, so a second run adds none.""" + + def find_spec(self, name, path, target=None): + if name not in PATCHES: + return None + for finder in sys.meta_path: + if getattr(finder, MARK, False) or not hasattr(finder, "find_spec"): + continue + spec = finder.find_spec(name, path, target) + if spec is not None: + break + else: + return None + if spec.loader is not None and hasattr(spec.loader, "exec_module"): + spec.loader = _Loader(spec.loader) + return spec + + +setattr(_Floor, MARK, True) +PATCHES = {DESKTOP_ENV: floor_desktop_env, EPISODE: stop_when_asked, DOCKER_PROVIDER: queue_starts} + + +def wrap_desktop_env(): + """A module of PATCHES already loaded is wrapped, and the finder goes on for its later loads.""" + for name in PATCHES: + if name in sys.modules: + PATCHES[name](sys.modules[name]) + if not any(getattr(f, MARK, False) for f in sys.meta_path): + sys.meta_path.insert(0, _Floor()) + + +def hide_proxy_password(environ=os.environ): + """Once a process, as wrap_docker_create.""" + try: + with open(environ["PROXY_CONFIG_FILE"], encoding="utf-8") as f: + words = {p["password"] for p in json.load(f) if isinstance(p.get("password"), str)} - {""} + except Exception: + return + plain = logging.Formatter.format + if not words or getattr(plain, MARK, False): + return + + @functools.wraps(plain) + def hidden(self, record): + text = plain(self, record) + for word in words: + text = text.replace(word, HIDDEN) + return text + + setattr(hidden, MARK, True) + logging.Formatter.format = hidden + logging.raiseExceptions = False + + +if __name__ == "sitecustomize": + hide_proxy_password() +if __name__ == "sitecustomize" and os.environ.get("NULLIUS_OPTION_VM_PROVIDER", "docker") == "docker": + wrap_docker_create() + wrap_desktop_env() +if __name__ == "__main__": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + if not sys.stdin.buffer.read(): + sweep(sys.argv[1]) diff --git a/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json b/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json new file mode 100644 index 0000000..3c82282 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/upstream-settings.json @@ -0,0 +1,835 @@ +[ + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--base_url", + "values": "an Anthropic-compatible endpoint", + "default": "$MINIMAX_ANTHROPIC_BASE_URL, else $ANTHROPIC_BASE_URL, else MiniMax's endpoint for MINIMAX_REGION", + "nullius": "fixed:the unit's relay, without /v1 - every model call goes through the relay, which holds the key; its documented invocation passes --base_url too" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--api_key", + "values": "a key", + "default": "$MINIMAX_API_KEY, else $ANTHROPIC_API_KEY", + "nullius": "fixed:the relay's placeholder - the provider's key stays in the relay; passed so that no MINIMAX_API_KEY from a .env file comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--extra_headers", + "values": "a JSON object of headers", + "default": "$ANTHROPIC_EXTRA_HEADERS, else unset", + "nullius": "fixed:unset - its default; a gateway's routing headers, which the relay's providers do not need" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--path_to_vm", + "values": "a VM image", + "default": "unset: the docker manager downloads it into ./docker_vm_data", + "nullius": "fixed:the upstream's docker_vm_data/osworld-v2-ubuntu-x86.qcow2, through a link in the unit's folder - the image its docker manager downloads, put there once before a run; its containers are found by that link" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--headless", + "values": "on, off", + "default": "off", + "nullius": "fixed:on - its documented invocation; the docker provider ignores it" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--action_space", + "values": "pyautogui", + "default": "pyautogui", + "nullius": "fixed:pyautogui - its default and its documented invocation; M3's parser writes pyautogui code" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--observation_type", + "values": "screenshot, a11y_tree, screenshot_a11y_tree, som", + "default": "screenshot", + "nullius": "fixed:screenshot - its default and its documented invocation; M3Agent reads screenshots" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--provider_name", + "values": "aws, virtualbox, vmware, docker, azure", + "default": "aws (its documented invocation: $PROVIDER_NAME, else aws)", + "nullius": "option:vm_provider" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--client_password", + "values": "a password", + "default": "empty: DesktopEnv then uses osworld-public-evaluation", + "nullius": "fixed:osworld-public-evaluation - its documented invocation's CLIENT_PASSWORD, the v2 image's password" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--screen_width", + "values": "pixels", + "default": "1920", + "nullius": "fixed:1920 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--screen_height", + "values": "pixels", + "default": "1080", + "nullius": "fixed:1080 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--sleep_after_execution", + "values": "seconds", + "default": "3.0", + "nullius": "fixed:3.0 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_steps", + "values": "a number", + "default": "100", + "nullius": "option:max_steps" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--checkpoint_eval_mode", + "values": "off, inline", + "default": "off", + "nullius": "fixed:off - its default: one evaluation, after the last step, written to result.txt" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--checkpoint_steps", + "values": "comma-separated step numbers", + "default": "empty", + "nullius": "fixed:empty - its default; read only with inline checkpoints" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--trace_guest (--trace_guest_memory)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; a debugging trace of the VM's resources" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--guest_trace_top_n (--guest_memory_top_n)", + "values": "a number", + "default": "30", + "nullius": "fixed:30 - its default; read only with --trace_guest" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--guest_trace_timeout (--guest_memory_trace_timeout)", + "values": "seconds", + "default": "15", + "nullius": "fixed:15 - its default; read only with --trace_guest" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--preserve_problem_machine", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; the aws provider's, which is not offered" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--enable_vnc (--disable_vnc)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; a viewer for a person watching the VM" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--enable_recording (--disable_recording)", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; no recording.mp4 is kept" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_trajectory_length", + "values": "a number", + "default": "10", + "nullius": "fixed:10 - its default, the screenshots M3Agent keeps in its history" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--coordinate_type", + "values": "relative, absolute", + "default": "relative", + "nullius": "fixed:relative - its default: M3's [0, 1000] coordinates scaled to the screen" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--thinking_mode", + "values": "adaptive, disabled, unset", + "default": "$M3_THINKING_MODE, else unset", + "nullius": "option:thinking" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_THINKING_MODE", + "values": "adaptive, disabled, empty", + "default": "empty in its documented invocation", + "nullius": "option:thinking" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--thinking_budget", + "values": "a number of tokens", + "default": "$M3_THINKING_BUDGET, else unset", + "nullius": "fixed:unset - its documented invocation leaves M3_THINKING_BUDGET empty; a budget turns thinking on by itself" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_THINKING_BUDGET", + "values": "a number of tokens", + "default": "empty in its documented invocation", + "nullius": "fixed:empty - its documented invocation's; set, it turns thinking on with that budget whatever the thinking option says" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_config_base_dir", + "values": "a folder", + "default": "evaluation_examples", + "nullius": "fixed:the upstream's evaluation_examples - its default, by its absolute path since each unit runs in its own folder" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--model", + "values": "a model name", + "default": "$MINIMAX_MODEL, else $ANTHROPIC_MODEL, else MiniMax-M3", + "nullius": "setup:original" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_MODEL, ANTHROPIC_MODEL", + "values": "a model name", + "default": "unset", + "nullius": "fixed:unset - --model names the setup's model" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--temperature", + "values": "a number", + "default": "0.6", + "nullius": "fixed:0.6 - its default (M3Agent sends 1 when thinking is on)" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--top_p", + "values": "a number", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--max_tokens", + "values": "a number", + "default": "8192", + "nullius": "fixed:8192 - its default and its documented invocation's MAX_TOKENS" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--stop_token", + "values": "a string", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--domain", + "values": "all, or one key of the meta file", + "default": "all", + "nullius": "fixed:all - its default; a unit's meta file names its one task" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_all_meta_path", + "values": "evaluation_examples/test_v2.json (108 tasks), test_v2_offline_no_internet.json (82), or one of ten lists by capability, each a subset of the 108 (conflict_disambiguation, cross_source_reasoning, dynamic_environment, human_in_the_loop, implicit_state_inference, multi_item_state_tracking, multimodal_editing, streaming_interaction, tutorial_following, visual_spatial_precision)", + "default": "evaluation_examples/test_v2.json", + "nullius": "dataset:test_v2" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--test_all_meta_path evaluation_examples/test_v2_offline_no_internet.json", + "values": "the upstream's offline list, 82 tasks", + "default": "-", + "nullius": "dataset:test_v2_offline_no_internet" + }, + { + "entry": "evaluation_examples/test_v2_offline_no_internet.json", + "setting": "its tasks that need no service", + "values": "36 of its 82: no mock website, GitLab, Moodle, LLM grader or simulated user", + "default": "-", + "nullius": "dataset:offline36" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need no hosted service", + "values": "63 of its 108: no mock website, GitLab or proxy; the LLM grader, the simulated user and the public internet allowed", + "default": "-", + "nullius": "dataset:no_services63" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need the mock websites", + "values": "35 of its 108: the mock websites, the LLM grader, the simulated user and the public internet allowed; no GitLab or proxy", + "default": "-", + "nullius": "dataset:websites35" + }, + { + "entry": "evaluation_examples/test_v2.json", + "setting": "its tasks that need no GitLab or proxy", + "values": "98 of its 108: No hosted services and the 35 that need the mock websites", + "default": "-", + "nullius": "dataset:websites98" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--eval_version", + "values": "v1, v2", + "default": "v2", + "nullius": "fixed:v2 - its default: the task classes of evaluation_examples/task_class, not OSWorld's v1 examples" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--specific_task_id", + "values": "a task id", + "default": "unset", + "nullius": "fixed:unset - its default; a unit's meta file names its one task" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--result_dir", + "values": "a folder", + "default": "./results", + "nullius": "fixed:results in the unit's folder - each trial keeps its own" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--num_envs", + "values": "a number", + "default": "1", + "nullius": "fixed:1 - its default and its documented invocation; Nullius runs units side by side (-j), each in a VM of its own" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--env_start_delay", + "values": "seconds", + "default": "1.0", + "nullius": "fixed:1.0 - its default; one environment, so no stagger happens" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--log_level", + "values": "DEBUG, INFO, WARNING, ERROR, CRITICAL", + "default": "INFO", + "nullius": "fixed:INFO - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--region", + "values": "an AWS region", + "default": "us-east-1 (its documented invocation: $AWS_DEFAULT_REGION)", + "nullius": "not offered - the aws provider's, which is not offered" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--snapshot_name", + "values": "a snapshot", + "default": "init_state", + "nullius": "fixed:init_state - its default; read by the vmware and virtualbox providers only" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--os_type", + "values": "Ubuntu, Windows", + "default": "Ubuntu", + "nullius": "fixed:Ubuntu - its default; the image is the v2 Ubuntu one" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--use_public_ip", + "values": "on, off", + "default": "off (its documented invocation: on when USE_PUBLIC_IP=1)", + "nullius": "fixed:off - its default; cloud providers only" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "--save_model_eval_raw_info", + "values": "on, off", + "default": "off", + "nullius": "fixed:off - its default; the LLM grader's inputs, and the grader is not set up" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_ANTHROPIC_BASE_URL, ANTHROPIC_BASE_URL", + "values": "an endpoint", + "default": "unset", + "nullius": "fixed:ANTHROPIC_BASE_URL the unit's relay, MINIMAX_ANTHROPIC_BASE_URL unset - --base_url passes the relay and comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_API_KEY, ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN", + "values": "a key", + "default": "unset", + "nullius": "fixed:ANTHROPIC_API_KEY the relay's placeholder, the others unset - --api_key passes the placeholder and comes first" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "MINIMAX_REGION", + "values": "global_en, cn_zh", + "default": "global_en", + "nullius": "fixed:unset - it picks MiniMax's own endpoint, which --base_url replaces with the relay" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_IMAGE_FORMAT", + "values": "JPEG, PNG", + "default": "JPEG", + "nullius": "fixed:JPEG - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_IMAGE_QUALITY", + "values": "1 to 100", + "default": "90", + "nullius": "fixed:90 - its default" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "M3_MAX_LLM_RETRIES", + "values": "a number", + "default": "2", + "nullius": "fixed:2 - its default: a reply with no action is asked again twice at most" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": "DesktopEnv(enable_proxy=...)", + "values": "True, False", + "default": "True, hard-coded", + "nullius": "fixed:True - hard-coded in the runner: the 8 tasks whose class asks for a proxy (3 of Offline, none of Offline, no services) get the upstream's DataImpulse setup" + }, + { + "entry": "scripts/python/run_multienv_m3.py", + "setting": ".env (load_dotenv)", + "values": "a file of variables", + "default": "loaded when present in the script's folder or a folder above it, by the script and by desktop_env/controllers/{setup,website,gitlab,moodle}.py", + "nullius": "fixed:what a .env file there holds - load_dotenv never overrides a variable already set; the driver sets ANTHROPIC_BASE_URL, ANTHROPIC_API_KEY, M3_THINKING_MODE, M3_THINKING_BUDGET, the grader's and the simulated user's provider, model, base URL and key, and passes --base_url, --api_key and --model, so such a file can still set any variable it leaves unset, among them ANTHROPIC_EXTRA_HEADERS, MINIMAX_REGION, M3_IMAGE_FORMAT, M3_IMAGE_QUALITY and M3_MAX_LLM_RETRIES" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "MODEL, RESULT_DIR, PROVIDER_NAME, OBSERVATION_TYPE, ACTION_SPACE, MAX_STEPS, NUM_ENVS, MAX_TOKENS, CLIENT_PASSWORD, AWS_DEFAULT_REGION, USE_PUBLIC_IP", + "values": "the wrapper's own variables", + "default": "MiniMax-M3, ./results/osworld_v2/m3, aws, screenshot, pyautogui, 100, 1, 8192, osworld-public-evaluation, us-east-1, 0", + "nullius": "fixed:the arguments they become - agents.json passes those arguments, each a row of run_multienv_m3.py" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "AWS_REGION, AWS_SUBNET_ID, AWS_SECURITY_GROUP_ID, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_INSTANCE_NAME, ENABLE_TTL", + "values": "the aws provider's settings", + "default": "exported with the wrapper's own values", + "nullius": "not offered - the aws provider's, which is not offered" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OPENAI_API_KEY", + "values": "a key", + "default": "exported empty", + "nullius": "fixed:unset - M3 does not read it" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_CLIENT_PASSWORD", + "values": "a password", + "default": "osworld-public-evaluation", + "nullius": "fixed:unset - exported, and read by none of the code M3 runs; --client_password passes the same password" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "WEBSITE_HOST_SUFFIX", + "values": "the mock websites' host suffix: that of OSWorld-web at the tag osworld-v2.1 served for this machine's VMs (websites.py); none refuses the 40 tasks that need it before they run", + "default": "web.hku.icu", + "nullius": "option:websites" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "GITLAB_URL", + "values": "a GitLab server", + "default": "the wrapper's server", + "nullius": "option:gitlab" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "GITLAB_PRIVATE_TOKEN", + "values": "the GitLab server's root token", + "default": "an empty token", + "nullius": "fixed:the licence GITLAB_PRIVATE_TOKEN - a secret, the driver's alone, given to the processes of the 2 GitLab tasks" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_EVAL_MODEL_PROVIDER, OSWORLD_EVAL_MODEL_API_KEY_ENV, OSWORLD_EVAL_MODEL_NAME", + "values": "the LLM grader's provider, key and model", + "default": "anthropic, ANTHROPIC_API_KEY, claude-sonnet-4-6", + "nullius": "fixed:anthropic, OSWORLD_EVAL_MODEL_API_KEY the relay's placeholder, the benchmark's judge (anthropic/claude-sonnet-4.6, OpenRouter's name for claude-sonnet-4-6) - the grader asks the unit's judge route, which holds the key and records its calls apart from the agent's" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_USER_SIM_PROVIDER, OSWORLD_USER_SIM_API_KEY_ENV, OSWORLD_USER_SIM_MODEL", + "values": "the simulated user's provider, key and model", + "default": "anthropic, ANTHROPIC_API_KEY, claude-sonnet-4-6", + "nullius": "fixed:anthropic, OSWORLD_USER_SIM_API_KEY the relay's placeholder, claude-sonnet-4-6 - the simulated user asks the unit's own route, which sends that model on as anthropic/claude-sonnet-4.6, a side model of the Original setup" + }, + { + "entry": "scripts/bash/run_multienv_m3.sh", + "setting": "OSWORLD_FILE_BASE_URL", + "values": "a folder or a URL of the task files", + "default": "$OSWORLD_FILE_BASE_URL, else ./cache/osworld_v2_assets", + "nullius": "fixed:the upstream's cache/osworld_v2_assets, by its absolute path - the release's gated assets, put there before a run: a file a task names through asset() is copied from there into the unit's task cache, not downloaded; for 001, 006 and 010 a folder of the unit's own, with their mail profiles signed out (driver.py _assets)" + }, + { + "entry": "desktop_env/controllers/website.py", + "setting": "OSWORLD_WEBSITE_TLS_VERIFY", + "values": "0, 1", + "default": "0", + "nullius": "fixed:unset (0) - the stack serves plain HTTP, which the controller falls back to when its HTTPS probe fails" + }, + { + "entry": "desktop_env/controllers/moodle.py", + "setting": "MOODLE_API_URL, MOODLE_API_KEY", + "values": "a Moodle server and its key", + "default": "unset", + "nullius": "not offered - no Moodle is hosted: no task of osworld-v2.1 uses it" + }, + { + "entry": "desktop_env/evaluators/model_client.py", + "setting": "the other OSWORLD_EVAL_MODEL_* variables, OPENAI_BASE_URL, MINIMAX_OPENAI_BASE_URL, LLM_DEBUG", + "values": "the LLM grader's endpoint, sampling, retries and logging", + "default": "unset", + "nullius": "fixed:OSWORLD_EVAL_MODEL_BASE_URL the unit's judge route without /v1, the others unset - the Anthropic client adds /v1/messages; sampling, retries and logging stay the upstream's defaults" + }, + { + "entry": "desktop_env/user_simulator.py", + "setting": "the other OSWORLD_USER_SIM_* variables", + "values": "the simulated user's endpoint, sampling and logging", + "default": "unset", + "nullius": "fixed:OSWORLD_USER_SIM_BASE_URL the unit's relay without /v1, the others unset - sampling and logging stay the upstream's defaults" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "PROXY_CONFIG_FILE", + "values": "a proxy pool file", + "default": "evaluation_examples/settings/proxy/dataimpulse.json", + "nullius": "fixed:its default, whose username and password are placeholders - the 8 tasks that ask for a proxy get no working proxy, and by the code their setup can fail; not checked by a run" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "HF_REPO, HF_TOKEN", + "values": "a mirror of the public file cache, a token", + "default": "unset", + "nullius": "fixed:unset - its default; read only for a download from huggingface.co, and no task of Offline, no services downloads from there" + }, + { + "entry": "desktop_env/controllers/setup.py", + "setting": "OSWORLD_STRICT_SETUP_TRANSPORT", + "values": "1, unset", + "default": "unset", + "nullius": "fixed:unset - its default" + }, + { + "entry": "desktop_env/desktop_env.py", + "setting": "SCREEN_WIDTH, SCREEN_HEIGHT", + "values": "pixels", + "default": "1920, 1080", + "nullius": "fixed:unset - the runner passes --screen_width and --screen_height" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "image", + "values": "a Docker image", + "default": "happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9", + "nullius": "fixed:the upstream's, pinned by digest - pulled before a run, since the provider would pull it within its port lock" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "DISK_SIZE, RAM_SIZE, CPU_CORES", + "values": "a size, a size, a count", + "default": "32G (or a task's own volume size), 4G, 4", + "nullius": "fixed:the task's own volume size, at least 40G (startup/sitecustomize.py, as the upstream's aws provider gives), 4G, 4 - the rest hard-coded in the upstream" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "LOCK_TIMEOUT", + "values": "seconds", + "default": "10, hard-coded", + "nullius": "fixed:none (startup/sitecustomize.py) - every VM start waits for the lock, a restart inside DesktopEnv.reset() too; the driver also starts units' VMs one after another" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "docker.from_env() timeout", + "values": "seconds a Docker call", + "default": "60, docker-py's default", + "nullius": "fixed:120 once the client creates the VM's container (startup/sitecustomize.py), and kept for its later calls (the containers' list, the stop and the removal) - a daemon busy with neighbouring VMs can take longer to create one; a create the client still times out on is not sent again, and the container the daemon made with that create's label is started" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "_wait_for_vm_ready(timeout)", + "values": "seconds", + "default": "300, hard-coded", + "nullius": "fixed:300 - hard-coded" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "KVM", + "values": "/dev/kvm when present", + "default": "hardware acceleration when /dev/kvm exists", + "nullius": "fixed:the machine's - with /dev/kvm the VM is accelerated, without it the upstream runs it unaccelerated" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "device_write_bps", + "values": "bytes a second per device", + "default": "unset: no limit, the provider's containers.run passes none", + "nullius": "option:write_mbps" + }, + { + "entry": "desktop_env/providers/docker/provider.py", + "setting": "device_write_iops", + "values": "write operations a second per device", + "default": "unset: no limit, the provider's containers.run passes none", + "nullius": "option:write_iops" + }, + { + "entry": "desktop_env/providers/docker/manager.py", + "setting": "UBUNTU_X86_URL, HF_ENDPOINT", + "values": "the image's address; a mirror", + "default": "huggingface.co/datasets/xlangai/v2-image at 6e16459a, pinned", + "nullius": "fixed:the upstream's pinned image - fetched and unpacked into docker_vm_data before a run, since an interrupted unpacking leaves a broken image" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "the Anthropic client's max_retries, timeout", + "values": "a count, seconds", + "default": "4, 180 s", + "nullius": "fixed:4, 180 s - hard-coded in M3Agent" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "stop_sequences", + "values": "a list of strings", + "default": ", Perform the next action. Perform", + "nullius": "fixed:its default - the runner passes none" + }, + { + "entry": "mm_agents/m3/agent.py", + "setting": "system_date", + "values": "a date in the system prompt", + "default": "the task's own date when its class sets one, else the day M3Agent is created", + "nullius": "fixed:its default - the runner passes none, and creates M3Agent for each unit: the model sees the day the unit starts, unless the task sets its own (4 of the 108 tasks, 1 of Offline, no services)" + }, + { + "entry": "lib_run_single.py", + "setting": "time.sleep(60) after reset", + "values": "seconds", + "default": "60, hard-coded", + "nullius": "fixed:60 - hard-coded: a slow or loaded machine can show the agent a desktop that has not caught up" + }, + { + "entry": "task_loader.py", + "setting": "the task's class", + "values": "evaluation_examples/task_class/task_NNN.py", + "default": "downloaded by scripts/tools/download_osworld_v2_tasks.py", + "nullius": "fixed:the release's gated task classes - put in evaluation_examples/task_class before a run" + }, + { + "entry": "scripts/python/run_multienv.py", + "setting": "its agent harness", + "values": "PromptAgent, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_claude.py", + "setting": "its agent harness", + "values": "Claude, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_claude_code.py", + "setting": "its agent harness", + "values": "Claude Code, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_codex.py", + "setting": "its agent harness", + "values": "Codex, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_deepseek.py", + "setting": "its agent harness", + "values": "DeepSeek, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_gemini.py", + "setting": "its agent harness", + "values": "Gemini, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_glm.py", + "setting": "its agent harness", + "values": "GLM, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_gpt_response_api.py", + "setting": "its agent harness", + "values": "GPT (Responses API), run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_muse_code.py", + "setting": "its agent harness", + "values": "Muse Code, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/run_multienv_muse_spark.py", + "setting": "its agent harness", + "values": "Muse Spark, run as published", + "default": "-", + "nullius": "setup:original" + }, + { + "entry": "scripts/python/run_multienv_muse_spark.py", + "setting": "--max_steps", + "values": "a number", + "default": "15", + "nullius": "option:max_steps" + }, + { + "entry": "scripts/python/run_multienv_qwen.py", + "setting": "its agent harness", + "values": "Qwen, run as published", + "default": "-", + "nullius": "not offered - this entry runs M3 and Muse Spark alone so far" + }, + { + "entry": "scripts/python/dispatch_deepseek_fullset.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/resume_deepseek_queue.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/rerun_deepseek_setup.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/record_deepseek_provenance.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "scripts/python/summarize_deepseek.py", + "setting": "its batch tool", + "values": "run as published", + "default": "-", + "nullius": "not offered - a batch tool around run_multienv_deepseek.py, which is not offered" + }, + { + "entry": "run.py", + "setting": "its agent harness", + "values": "run as published", + "default": "-", + "nullius": "not offered - a single-environment runner of OSWorld's v1 examples" + }, + { + "entry": "scripts/python/manual_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/python/interactive_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/python/debug_examine.py", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_human_examine.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_interactive_examine.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_debug.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_manual_examine_windows.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "scripts/bash/run_mannual_examine_windows.sh", + "setting": "its harness", + "values": "a person at the VM", + "default": "manual", + "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "the paper (arXiv 2606.29537)", + "setting": "M3's run, Table 3", + "values": "500 steps, thinking on, 8K output tokens, AWS instances with all web traffic routed through a residential proxy, release osworld-v2-2026.06.24", + "default": "-", + "nullius": "setup:original" + } +] diff --git a/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml b/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml new file mode 100644 index 0000000..f75e401 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/websites.compose.yml @@ -0,0 +1,176 @@ +name: osworld2-web + +x-blkio: &blkio + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_QUIET_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_QUIET_IOPS:?as websites.py sets it} + +x-logging: &logs + driver: json-file + options: + max-size: 10m + max-file: "3" + +services: + awsconsole: + blkio_config: *blkio + logging: *logs + budgetwise_web_backend: + blkio_config: *blkio + logging: *logs + budgetwise_web_frontend: + blkio_config: *blkio + logging: *logs + caddy: + image: lucaslorentz/caddy-docker-proxy@sha256:33081120c6df8613cebbdbdd4c3b8aef641e9e56c1cd3d86d04ea9651cb3cc68 + ports: !override + - ${OSWORLD2_WEB_ADDRESS:?the address Caddy listens on, as websites.py sets it}:80:80 + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CADDY_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CADDY_IOPS:?as websites.py sets it} + logging: *logs + calendar_web: + blkio_config: *blkio + logging: *logs + careerlink_web: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CAREERLINK_WEB_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_CAREERLINK_WEB_IOPS:?as websites.py sets it} + logging: *logs + cloudcrm_web_backend: + blkio_config: *blkio + logging: *logs + cloudcrm_web_frontend: + blkio_config: *blkio + logging: *logs + dinogame_web: + blkio_config: *blkio + logging: *logs + eventix_web_backend: + blkio_config: *blkio + logging: *logs + eventix_web_frontend: + blkio_config: *blkio + logging: *logs + expenseflow_web_backend: + blkio_config: *blkio + logging: *logs + expenseflow_web_frontend: + blkio_config: *blkio + logging: *logs + formcraft_web_backend: + blkio_config: *blkio + logging: *logs + formcraft_web_frontend: + blkio_config: *blkio + logging: *logs + glbviewer_web: + blkio_config: *blkio + logging: *logs + insurance_claim_web_backend: + blkio_config: *blkio + logging: *logs + insurance_claim_web_frontend: + blkio_config: *blkio + logging: *logs + interfacinglinux_web: + blkio_config: *blkio + logging: *logs + mailhub_web_backend: + blkio_config: *blkio + logging: *logs + mailhub_web_frontend: + blkio_config: *blkio + logging: *logs + overleaf_collab_compiler: + blkio_config: *blkio + logging: *logs + overleaf_collab_web: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_COLLAB_WEB_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_COLLAB_WEB_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_admin_bootstrap: + blkio_config: *blkio + logging: *logs + overleaf_web_app: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_APP_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_APP_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_mongo: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_MONGO_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_MONGO_IOPS:?as websites.py sets it} + logging: *logs + overleaf_web_mongo_init: + blkio_config: *blkio + logging: *logs + overleaf_web_redis: + blkio_config: + device_write_bps: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_REDIS_BPS:?as websites.py sets it} + device_write_iops: + - path: ${OSWORLD2_WEB_DISK:?the disk under Docker's root, as websites.py sets it} + rate: ${OSWORLD2_WEB_OVERLEAF_WEB_REDIS_IOPS:?as websites.py sets it} + logging: *logs + reviewsphere: + blkio_config: *blkio + logging: *logs + slidepuzzle_web: + blkio_config: *blkio + logging: *logs + streamview_web_backend: + blkio_config: *blkio + logging: *logs + streamview_web_frontend: + blkio_config: *blkio + logging: *logs + teamchat_web_backend: + blkio_config: *blkio + logging: *logs + teamchat_web_frontend: + blkio_config: *blkio + logging: *logs + travelhub_pro_web: + blkio_config: *blkio + logging: *logs + trippza_web: + blkio_config: *blkio + logging: *logs + vaultbank_web_backend: + blkio_config: *blkio + logging: *logs + vaultbank_web_frontend: + blkio_config: *blkio + logging: *logs + visaapplication_web: + blkio_config: *blkio + logging: *logs + wandb: + blkio_config: *blkio + logging: *logs diff --git a/app/nullius/catalogue/benchmarks/osworld2/websites.py b/app/nullius/catalogue/benchmarks/osworld2/websites.py new file mode 100644 index 0000000..ea19dae --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld2/websites.py @@ -0,0 +1,668 @@ +"""OSWorld2's mock websites: Task-Web/OSWorld-web at the release's tag, as its tasks reach them. + +The sites are OSWorld-web's compose project at COMMIT with websites.compose.yml +over it; tasks of TASKS reach them at ., the run's websites +option. Caddy listens on the VMs' network's gateway only. Overleaf keeps no +slice per task, so its tasks run one at a time under LOCK, on fresh volumes; +a unit that needs the sites holds SITES_LOCK shared, reset, smoke and compose's changes alone. + +python3 websites.py configure STACK SUFFIX --write-mbps N --write-iops N +python3 websites.py builder STACK up --write-mbps N --write-iops N | down +python3 websites.py build STACK [--group NAME]... [--reserve-gb 25] +python3 websites.py compose STACK ARGS... +python3 websites.py check SUFFIX | smoke STACK | reset STACK""" +import argparse +import contextlib +import errno +import fcntl +import importlib.util +import json +import os +import re +import shutil +import socket +import stat +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +OVERRIDE = os.path.join(HERE, "websites.compose.yml") +SITE = os.path.join(HERE, "startup", "sitecustomize.py") +COMMIT = "60c89fe6a8ed934668619d8d26132848239eb8ee" +PROJECT = "osworld2-web" +SCHEME = "http://" +OPTION = "NULLIUS_OPTION_WEBSITES" +SETUP_DOC = "docs/benchmarks/osworld2-machine.md" +STATEFUL = ("awsconsole", "budgetwise", "calendar", "careerlink", "cloudcrm", "dinogame", "eventix", "expenseflow", + "formcraft", "glbviewer", "insurance-claim", "mailhub", "overleaf-collab", "reviewsphere", "slidepuzzle", + "streamview", "teamchat", "travelhubpro", "trippza", "vaultbank", "visaapplication", "wandb") +NAMES = STATEFUL + ("studio.streamview", "interfacinglinux", "overleaf") +TASKS = frozenset("005 007 008 013 015 016 018 019 020 021 026 028 031 035 036 038 039 043 046 050 052 056 057 058 " + "060 065 068 069 070 072 073 077 079 081 082 091 098 100 101 102".split()) +OVERLEAF_TASKS = frozenset(("057", "072")) +OVERLEAF = ("overleaf_web_mongo", "overleaf_web_mongo_init", "overleaf_web_redis", "overleaf_web_admin_bootstrap", + "overleaf_web_app") +OVERLEAF_VOLUMES = ("overleaf_web_data", "overleaf_web_mongo_data", "overleaf_web_redis_data") +LOCK = "/tmp/nullius-osworld2-overleaf.lck" +SITES_LOCK = "/tmp/nullius-osworld2-websites.lck" +READING = frozenset(("config", "ps", "ls", "logs", "images", "top", "port", "events", "version", "stats", "wait")) +VALUED = frozenset(("-f", "--file", "-p", "--project-name", "--project-directory", "--profile", "--env-file", + "--ansi", "--progress", "--parallel")) +COMPOSE_VERSION = (2, 24, 4) +_HELD = [] +_SITE = [] +COOKIE = "nullius-osworld2-ready" +SUFFIX = re.compile(r"(?=.{3,200}$)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+") +GROUPS = (("pulled", ("caddy", "overleaf_web_mongo", "overleaf_web_mongo_init", "overleaf_web_redis", + "overleaf_web_admin_bootstrap"), 7), + ("overleaf", ("overleaf_web_app", "overleaf_collab_compiler"), 12), + ("next-a", ("awsconsole", "calendar_web", "careerlink_web", "dinogame_web", "glbviewer_web", "reviewsphere"), 9), + ("next-b", ("slidepuzzle_web", "travelhub_pro_web", "trippza_web", "visaapplication_web", "wandb", + "overleaf_collab_web"), 9), + ("frontends", ("budgetwise_web_frontend", "cloudcrm_web_frontend", "eventix_web_frontend", + "expenseflow_web_frontend", "formcraft_web_frontend", "mailhub_web_frontend", + "streamview_web_frontend", "teamchat_web_frontend", "vaultbank_web_frontend", + "insurance_claim_web_frontend", "interfacinglinux_web"), 7), + ("backends", ("budgetwise_web_backend", "cloudcrm_web_backend", "eventix_web_backend", + "expenseflow_web_backend", "formcraft_web_backend", "insurance_claim_web_backend", + "mailhub_web_backend", "streamview_web_backend", "teamchat_web_backend", + "vaultbank_web_backend"), 4)) +SERVICES = tuple(s for _, group, _ in GROUPS for s in group) +WEIGHTS = {"overleaf_web_mongo": (30, 10), "overleaf_web_app": (30, 6), "overleaf_web_redis": (10, 4), + "overleaf_collab_web": (46, 22), "careerlink_web": (6, 2), "caddy": (4, 2)} +QUIET = (1, 1) +TIERS = ("QUIET",) + tuple(s.upper() for s in WEIGHTS) +SETTINGS = ("HOST_SUFFIX", "CADDY_SCHEME", "OSWORLD2_WEB_ADDRESS", "OSWORLD2_WEB_DISK") + tuple( + f"OSWORLD2_WEB_{t}_{k}" for t in TIERS for k in ("BPS", "IOPS")) +MB = 1024 * 1024 +BUILDER = "osw2-web" +BUILDKIT = "moby/buildkit@sha256:de10faf919fc71ba4eb1dd7bd6449566d012b0c9436b1c61bfee21d621b009aa" +BUILDKITD = "nullius-osworld2-buildkitd" +BUILDKIT_VOLUME = "nullius-osworld2-buildkit" +BUILDKIT_PARALLEL = 4 +RESERVE_GB = 25 +IMAGES_FILE = ".osworld2.images.json" +BUILDKIT_FILE = ".osworld2.buildkitd.toml" +ENV_FILE = ".osworld2.env" +COMPOSE_FILE = ".osworld2.compose.yml" +LABEL = "com.docker.compose." +CADDY = re.compile(r"^\s+image: (lucaslorentz/caddy-docker-proxy@sha256:[0-9a-f]{64})$", re.M) +PROBE = "nullius-osworld2-web-probe-" +HTTP_SECONDS = 10 +DOCKER_SECONDS = 300 +OVERLEAF_SECONDS = 600 +RESET_SECONDS = 900 +BUILD_SECONDS = 4 * 3600 +POLL = 5 +PAGE_BYTES = 1 << 20 +PROXIES = {} +START = "start the stack (python3 websites.py compose STACK up -d) and check it (python3 websites.py check SUFFIX)" + + +class WebsiteError(RuntimeError): + pass + + +def suffix(environ=os.environ): + """The run's websites option, the sites' host suffix; None for none.""" + value = environ.get(OPTION, "none") + if value == "none": + return None + if not SUFFIX.fullmatch(value): + raise WebsiteError(f"the websites option {value!r} is not a host suffix: give the suffix the stack serves " + f"(such as {_site().GATEWAY}.nip.io), or none") + return value + + +def env(environ=os.environ): + """The variables the upstream reads the sites by, for every process the driver starts.""" + value = suffix(environ) + return {"WEBSITE_HOST_SUFFIX": value} if value else {} + + +def needed(eid, environ=os.environ): + """A task of TASKS without a suffix is refused, since its class cannot even load.""" + value = suffix(environ) + if eid not in TASKS: + return None + if not value: + raise WebsiteError(f"task {eid} needs the mock websites and this run gives it none: set the websites option " + f"to the suffix the stack serves ({SETUP_DOC})") + return value + + +def _resolves(host): + try: + return {a[4][0] for a in socket.getaddrinfo(host, 80, type=socket.SOCK_STREAM)} + except OSError: + return set() + + +def _fetch(opener, url): + """Any failure, a status of 400 or more among them, is a WebsiteError.""" + try: + with opener.open(urllib.request.Request(url, headers={"User-Agent": "nullius-osworld2"}), + timeout=HTTP_SECONDS) as r: + return r.read(PAGE_BYTES) + except urllib.error.HTTPError as e: + e.close() + raise WebsiteError(f"{url} answered {e.code}: {START}") from None + except (urllib.error.URLError, OSError, ValueError) as e: + reason = getattr(e, "reason", e) + raise WebsiteError(f"{url} did not answer ({type(reason).__name__}: {reason}): {START}") from None + + +def ready(value, overleaf=False, state=True): + """The number of names checked; Overleaf's only with ``overleaf``, each /api/state only with ``state``.""" + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + names = NAMES if overleaf else tuple(n for n in NAMES if n != "overleaf") + gateway = _site().GATEWAY + for name in names: + host = f"{name}.{value}" + found = _resolves(host) + if not found: + raise WebsiteError(f"{host} does not resolve: the websites option must be a wildcard DNS name of the " + f"address the stack listens on, {gateway} (such as {gateway}.nip.io)") + if gateway not in found: + raise WebsiteError(f"{host} resolves to {', '.join(sorted(found))}, not {gateway}, the VMs' network's " + f"gateway the stack listens on: set the websites option to a wildcard DNS name of " + f"{gateway} (such as {gateway}.nip.io)") + if not _fetch(opener, f"{SCHEME}{host}/").strip(): + raise WebsiteError(f"{SCHEME}{host}/ answered an empty page: the stack serves another suffix than " + f"{value}, or not this site; {START}") + for name in STATEFUL if state else (): + url = f"{SCHEME}{name}.{value}/api/state?cookie={COOKIE}" + try: + said = json.loads(_fetch(opener, url)) + except ValueError: + said = None + if not isinstance(said, dict) or said.get("user_id") != COOKIE or not isinstance(said.get("state"), dict): + raise WebsiteError(f"{url} answered no state of the site's: {START}") + return len(names) + + +def _docker(*args, seconds=DOCKER_SECONDS): + try: + return subprocess.run(["docker", *args], capture_output=True, text=True, timeout=seconds, + env=_clean(), stdin=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + raise WebsiteError(f"docker {args[0]} got no answer in {seconds} s: is its daemon running?") from None + except OSError: + raise WebsiteError("no docker on this machine's PATH: the mock websites run on Docker") from None + + +def _clean(): + """Without the stack's settings, so compose reads them from its environment file alone.""" + return {k: v for k, v in os.environ.items() if k not in SETTINGS} + + +def _done(said, what): + if said.returncode != 0: + last = ((said.stderr or "").strip().splitlines() or ["no reason"])[-1] + raise WebsiteError(f"{what} failed: {last[:300]}") + return said + + +def _settings(path): + try: + with open(path, encoding="utf-8") as f: + return dict(ln.rstrip("\n").partition("=")[::2] for ln in f if "=" in ln) + except OSError as e: + raise WebsiteError(f"no settings of the stack at {path} ({type(e).__name__}): run python3 websites.py " + "configure first") from None + + +def _project(): + """The running project's compose command and settings, from any of its containers' labels.""" + fields = ("project.config_files", "project.working_dir", "project.environment_file") + said = _docker("ps", "-a", "--filter", f"label={LABEL}project={PROJECT}", + "--format", "\t".join('{{.Label "' + LABEL + f + '"}}' for f in fields)) + rows = [ln.split("\t") for ln in said.stdout.splitlines() if ln.strip()] if said.returncode == 0 else [] + if not rows: + raise WebsiteError(f"no container of the {PROJECT} project on this machine: {START}") + files, where, envfile = (rows[0] + ["", "", ""])[:3] + if not envfile or not files: + raise WebsiteError(f"the {PROJECT} project was not started by websites.py compose, which gives it its " + "settings: start it that way") + return (["compose", "-p", PROJECT, "--project-directory", where, + *[x for f in files.split(",") for x in ("-f", f)], "--env-file", envfile], _settings(envfile)) + + +def _waited(url, seconds, closed, what): + opener = urllib.request.build_opener(urllib.request.ProxyHandler(PROXIES)) + t0, last = time.time(), "no answer" + while True: + try: + if _fetch(opener, url).strip(): + return + last = "an empty page" + except WebsiteError as e: + last = str(e).split(": ", 1)[0] + if time.time() - t0 >= seconds: + raise WebsiteError(f"{what} did not come up in {seconds} s ({last})") + if closed is None: + time.sleep(POLL) + elif closed.wait(POLL): + raise WebsiteError(f"{what} was not up when the unit's time ran out") + + +def reset_overleaf(value, closed=None): + """Overleaf's services removed with their volumes, started again, and waited on until its login answers.""" + compose, settings = _project() + if settings.get("HOST_SUFFIX") != value: + raise WebsiteError(f"the stack serves {settings.get('HOST_SUFFIX') or 'no suffix'} and this run asks for " + f"{value}: set the websites option to the stack's") + _done(_docker(*compose, "rm", "--stop", "--force", "-v", *OVERLEAF), "removing Overleaf's services") + _done(_docker("volume", "rm", "--force", *[f"{PROJECT}_{v}" for v in OVERLEAF_VOLUMES]), "removing Overleaf's volumes") + _done(_docker(*compose, "up", "-d", "--no-build", "--pull", "never", OVERLEAF[-1]), "starting Overleaf") + _waited(f"{SCHEME}overleaf.{value}/login", OVERLEAF_SECONDS, closed, "Overleaf") + + +def opened(path): + """Read-only, so another user's lock in /tmp serves too; kept only as a plain file, since a FIFO would hang the open.""" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + while True: + try: + fd = os.open(path, flags) + break + except FileNotFoundError: + pass + try: + fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o644) + os.fchmod(fd, 0o644) + break + except FileExistsError: + pass + if not stat.S_ISREG(os.fstat(fd).st_mode): + os.close(fd) + raise OSError(errno.EINVAL, "not a plain file", path) + return fd + + +@contextlib.contextmanager +def locked(path, mode, closed, busy): + """Waited on until ``closed`` is set, or refused at once with no ``closed``; one file for every user of the machine.""" + try: + fd = opened(path) + except OSError as e: + raise WebsiteError(f"the lock {path} cannot be opened ({type(e).__name__}): remove it if it is not a plain " + "file every user can read") from None + try: + while True: + try: + fcntl.flock(fd, mode | fcntl.LOCK_NB) + break + except BlockingIOError: + if closed is None: + raise WebsiteError(busy) from None + if closed.wait(1): + raise WebsiteError(f"the unit's time ran out while {busy}") from None + _HELD.append(fd) + try: + yield + finally: + _HELD.remove(fd) + finally: + os.close(fd) + + +def held_fds(): + """For a process that keeps the locks held after this one lets go.""" + return tuple(_HELD) + + +@contextlib.contextmanager +def held(eid, value, closed): + """For a task of OVERLEAF_TASKS, Overleaf alone and anew.""" + if not value: + yield + return + with locked(SITES_LOCK, fcntl.LOCK_SH, closed, "the stack was being reset or smoked"): + if eid not in OVERLEAF_TASKS: + yield + return + with locked(LOCK, fcntl.LOCK_EX, closed, "another unit's Overleaf task ran"): + reset_overleaf(value, closed) + yield + + +def _alone(): + return locked(SITES_LOCK, fcntl.LOCK_EX, None, "a unit is using the mock websites: reset or smoke the stack " + "when no task that needs them runs") + + +def _override(): + try: + with open(OVERRIDE, encoding="utf-8") as f: + return f.read() + except OSError as e: + raise WebsiteError(f"no {OVERRIDE} ({type(e).__name__}): this benchmark's folder is incomplete; " + "install Nullius again") from None + + +def _compose_version(): + """Docker Compose COMPOSE_VERSION or later, which websites.compose.yml's !override needs.""" + said = _docker("compose", "version", "--short") + found = re.match(r"v?(\d+)\.(\d+)\.(\d+)", said.stdout.strip()) if said.returncode == 0 else None + if not found or tuple(int(x) for x in found.groups()) < COMPOSE_VERSION: + raise WebsiteError("websites.compose.yml needs Docker Compose " + ".".join(map(str, COMPOSE_VERSION)) + + f" or later (its !override); this machine has {said.stdout.strip() or 'none'}") + + +def _pinned(stack): + """``stack`` is OSWorld-web at COMMIT, its submodules at the commits it names, nothing tracked changed.""" + def git(*args): + try: + return subprocess.run(["git", "-C", stack, *args], capture_output=True, text=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as e: + raise WebsiteError(f"git could not read {stack}: {type(e).__name__}") from None + head = git("rev-parse", "HEAD") + if head.returncode != 0 or head.stdout.strip() != COMMIT: + raise WebsiteError(f"{stack} is not OSWorld-web at {COMMIT} (the tag osworld-v2.1): clone it with its " + f"submodules at that tag ({SETUP_DOC})") + modules = git("submodule", "status") + off = [ln.split()[1] for ln in modules.stdout.splitlines() if ln[:1] != " " and len(ln.split()) > 1] + if modules.returncode != 0 or off: + raise WebsiteError(f"{stack}'s submodules are not at the commits it names ({', '.join(off) or 'unreadable'}): " + "git submodule update --init --recursive") + changed = git("status", "--porcelain", "--untracked-files=no") + if changed.returncode != 0 or changed.stdout.strip(): + raise WebsiteError(f"{stack} has changes to its files: the stack runs as published") + return os.path.abspath(stack) + + +def _beside(stack, name): + return os.path.abspath(stack).rstrip(os.sep) + name + + +def _write(path, text): + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".nullius-") + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(text) + os.chmod(tmp, 0o644) + os.replace(tmp, path) + + +def allocation(mbps, iops): + """Each tier's per-service limit by weight, rounded down, so all services together stay within the share.""" + if mbps < 1 or iops < 1: + raise WebsiteError("the stack's write share is whole numbers above 0") + weights = [WEIGHTS.get(s, QUIET) for s in SERVICES] + total = [sum(w[i] for w in weights) for i in (0, 1)] + out = {} + for tier, (wb, wi) in [("QUIET", QUIET)] + [(s.upper(), w) for s, w in WEIGHTS.items()]: + out[tier] = (mbps * MB * wb // total[0], iops * wi // total[1]) + if not all(out[tier]): + raise WebsiteError(f"a write share of {iops} IOPS leaves the {tier.lower()} services none: give the " + f"stack at least {total[1]} IOPS") + return out + + +def limits(settings): + """Every service's limit as the settings give it, and their sum.""" + each = {s: (int(settings[f"OSWORLD2_WEB_{t}_BPS"]), int(settings[f"OSWORLD2_WEB_{t}_IOPS"])) + for s in SERVICES for t in [s.upper() if s in WEIGHTS else "QUIET"]} + return each, tuple(sum(v[i] for v in each.values()) for i in (0, 1)) + + +def _site(): + """Loaded under another name, so it wraps nothing in this process.""" + if not _SITE: + spec = importlib.util.spec_from_file_location("osworld2_startup", SITE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _SITE.append(module) + return _SITE[0] + + +def _network(): + """Made when missing; its gateway is the address Caddy listens on.""" + site = _site() + try: + return site.network(_docker) + except site.NetworkError as e: + raise WebsiteError(str(e)) from None + + +def _root_disk(): + info = _done(_docker("info", "--format", "{{.DockerRootDir}}"), "docker info") + site = _site() + try: + return info.stdout.strip(), site.root_disk(info.stdout.strip()) + except site.WriteLimitError as e: + raise WebsiteError(f"the write limit cannot be applied: {e}") from None + + +def configure(stack, value, mbps, iops): + """The stack's settings: Caddy on the VMs' network's gateway, each tier with its write limit.""" + stack = _pinned(stack) + if not SUFFIX.fullmatch(value): + raise WebsiteError(f"{value!r} is not a host suffix") + tiers = allocation(mbps, iops) + _network() + settings = {"HOST_SUFFIX": value, "CADDY_SCHEME": SCHEME, "OSWORLD2_WEB_ADDRESS": _site().GATEWAY, + "OSWORLD2_WEB_DISK": _root_disk()[1], + **{f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))}} + path = _beside(stack, ENV_FILE) + _write(path, "".join(f"{k}={settings[k]}\n" for k in SETTINGS)) + return path, settings + + +def compose(stack, args, capture=False, builder=None): + """Compose on the stack, websites.compose.yml copied beside it and the VMs' network made first.""" + stack = _pinned(stack) + envfile = _beside(stack, ENV_FILE) + _settings(envfile) + _compose_version() + _network() + copy = _beside(stack, COMPOSE_FILE) + _write(copy, _override()) + argv = ["docker", "compose", "-p", PROJECT, "--project-directory", stack, "-f", + os.path.join(stack, "docker-compose.yml"), "-f", copy, "--env-file", envfile, *args] + env = dict(_clean(), **({"BUILDX_BUILDER": builder} if builder else {})) + if capture: + return subprocess.run(argv, capture_output=True, text=True, env=env, stdin=subprocess.DEVNULL, + timeout=BUILD_SECONDS if builder else RESET_SECONDS, pass_fds=held_fds()) + return subprocess.run(argv, env=env, pass_fds=held_fds()).returncode + + +def builder(stack, op, mbps=0, iops=0): + """BuildKit, privileged as buildx's own builders, with its own write limit; down removes it with its cache.""" + if op == "down": + gone = [_docker("buildx", "rm", BUILDER), _docker("rm", "-f", BUILDKITD), _docker("volume", "rm", "--force", BUILDKIT_VOLUME)] + failed = [g for g in gone[1:] if g.returncode != 0] + if failed: + _done(failed[0], "removing the builder") + return {"builder": BUILDER, "removed": True} + if mbps < 1 or iops < 1: + raise WebsiteError("the builder's write limit is whole numbers above 0") + stack = _pinned(stack) + disk = _root_disk()[1] + config = _beside(stack, BUILDKIT_FILE) + _write(config, f"[worker.oci]\n max-parallelism = {BUILDKIT_PARALLEL}\n") + _done(_docker("run", "-d", "--name", BUILDKITD, "--privileged", "--restart", "no", + "--device-write-bps", f"{disk}:{mbps * MB}", "--device-write-iops", f"{disk}:{iops}", + "-v", f"{BUILDKIT_VOLUME}:/var/lib/buildkit", "-v", f"{config}:/etc/buildkit/buildkitd.toml:ro", + BUILDKIT), "starting the builder") + host = json.loads(_done(_docker("inspect", "-f", "{{json .HostConfig}}", BUILDKITD), "inspecting the builder").stdout) + kept = {"Path": disk, "Rate": mbps * MB} in (host.get("BlkioDeviceWriteBps") or []) and \ + {"Path": disk, "Rate": iops} in (host.get("BlkioDeviceWriteIOps") or []) + if not kept: + _docker("rm", "-f", BUILDKITD) + raise WebsiteError(f"Docker kept no write limit on {disk} for the builder: it is removed, nothing was built") + _done(_docker("buildx", "create", "--name", BUILDER, "--driver", "remote", f"docker-container://{BUILDKITD}"), + "registering the builder") + return {"builder": BUILDER, "disk": disk, "write_bps": mbps * MB, "write_iops": iops} + + +def _free_gb(root): + st = os.statvfs(root) + return st.f_bavail * st.f_frsize / 1e9 + + +def build(stack, groups=(), reserve=RESERVE_GB, say=print): + """A group that would leave less than ``reserve`` GB free stops the build; the cache is removed after each group.""" + stack = _pinned(stack) + names = [g[0] for g in GROUPS] + unknown = [g for g in groups if g not in names] + if unknown: + raise WebsiteError(f"no group {', '.join(unknown)}: the groups are {', '.join(names)}") + if _docker("buildx", "inspect", BUILDER).returncode != 0: + raise WebsiteError(f"no builder {BUILDER}: start it (python3 websites.py builder STACK up ...)") + root = _root_disk()[0] + done = [] + for name, services, estimate in GROUPS: + if groups and name not in groups: + continue + free = _free_gb(root) + if free - estimate < reserve: + raise WebsiteError(f"stopped before {name}: {free:.1f} GB free under {root}, it needs about {estimate} GB " + f"and {reserve} GB stay free (done: {', '.join(done) or 'none'})") + said = compose(stack, ["pull", *services] if name == "pulled" else ["build", *services], capture=True, + builder=BUILDER) + _done(said, f"the {name} group") + if name != "pulled": + _done(_docker("buildx", "prune", "--builder", BUILDER, "--all", "--force"), "removing the builder's cache") + done.append(name) + say(json.dumps({"group": name, "free_gb": round(_free_gb(root), 1)})) + config = json.loads(_done(compose(stack, ["config", "--format", "json"], capture=True), "reading the project").stdout) + ids = {} + for s in SERVICES: + ref = config["services"].get(s, {}).get("image") or f"{PROJECT}-{s}" + said = _docker("image", "inspect", "--format", "{{.Id}}", ref) + ids[s] = {"image": ref, "id": said.stdout.strip() if said.returncode == 0 else None} + path = _beside(stack, IMAGES_FILE) + _write(path, json.dumps({"commit": COMMIT, "groups": done, "images": ids}, indent=1) + "\n") + return {"groups": done, "images": path} + + +def _until_ready(value, seconds): + t0 = time.time() + while True: + try: + return ready(value, overleaf=True) + except WebsiteError: + if time.time() - t0 >= seconds: + raise + time.sleep(POLL) + + +def reset(stack): + value = _settings(_beside(os.path.abspath(stack), ENV_FILE)).get("HOST_SUFFIX", "") + with _alone(): + _done(compose(stack, ["down", "--volumes"], capture=True), "taking the stack down") + _done(compose(stack, ["up", "-d", "--no-build", "--pull", "never"], capture=True), "starting the stack") + return _until_ready(value, RESET_SECONDS) + + +def _from_vms(value): + """Each name from a container on the VMs' network, by Caddy's own busybox wget.""" + image = CADDY.search(_override()).group(1) + loop = 'for h; do wget -q -O /dev/null -T 10 "http://$h/" || { echo "$h"; exit 1; }; done' + network = _network() + said = _docker("run", "--rm", "--name", f"{PROBE}{os.getuid()}-{os.getpid()}", "--network", network, + "--pull", "never", "--entrypoint", "sh", image, "-c", loop, "sh", + *[f"{n}.{value}" for n in NAMES]) + if said.returncode != 0: + failed = (said.stdout.strip().splitlines() or [""])[-1] + raise WebsiteError(f"from the VMs' network {network}, {failed or 'a name'} did not answer: " + + ((said.stderr or "").strip().splitlines() or ["no reason"])[-1][:300]) + + +def smoke(stack): + stack = _pinned(stack) + value = _settings(_beside(stack, ENV_FILE)).get("HOST_SUFFIX", "") + with _alone(): + return _smoked(stack, value) + + +def _smoked(stack, value): + out = {"suffix": value, "names": ready(value, overleaf=True)} + _from_vms(value) + out["network"] = len(NAMES) + node = shutil.which("node") + if not node: + raise WebsiteError("no node on PATH: the upstream's scripts/verify-control-plane.mjs needs Node 18 or later") + script = os.path.join(stack, "scripts", "verify-control-plane.mjs") + said = subprocess.run([node, script], capture_output=True, text=True, timeout=RESET_SECONDS, stdin=subprocess.DEVNULL, + env=dict(_clean(), OSWORLD_HOST_SUFFIX=value, OSWORLD_SCHEME=SCHEME[:-3])) + passed = [ln.split()[1] for ln in said.stdout.splitlines() if ln.startswith("PASS ")] + if said.returncode != 0: + raise WebsiteError(f"the upstream's control-plane check failed after {len(passed)} sites: " + + ((said.stderr or said.stdout).strip().splitlines() or ["no reason"])[-1][:300]) + out["control_plane"] = len(passed) + reset_overleaf(value) + out["overleaf"] = "anew" + return out + + +def reads(args): + """Whether compose ``args`` only read: their subcommand is one of READING.""" + skip = False + for x in args: + if skip: + skip = False + elif x in VALUED: + skip = True + elif not x.startswith("-"): + return x in READING + return False + + +def main(argv): + p = argparse.ArgumentParser(prog="websites.py", description="OSWorld2's mock websites") + sub = p.add_subparsers(dest="op", required=True) + c = sub.add_parser("configure") + c.add_argument("stack") + c.add_argument("suffix") + c.add_argument("--write-mbps", type=int, required=True) + c.add_argument("--write-iops", type=int, required=True) + d = sub.add_parser("compose") + d.add_argument("stack") + d.add_argument("args", nargs=argparse.REMAINDER) + b = sub.add_parser("builder") + b.add_argument("stack") + b.add_argument("op_", metavar="up|down", choices=("up", "down")) + b.add_argument("--write-mbps", type=int, default=0) + b.add_argument("--write-iops", type=int, default=0) + g = sub.add_parser("build") + g.add_argument("stack") + g.add_argument("--group", action="append", default=[]) + g.add_argument("--reserve-gb", type=float, default=RESERVE_GB) + sub.add_parser("check").add_argument("suffix") + sub.add_parser("smoke").add_argument("stack") + sub.add_parser("reset").add_argument("stack") + a = p.parse_args(argv) + try: + if a.op == "compose": + args = a.args[1:] if a.args[:1] == ["--"] else a.args + with contextlib.nullcontext() if reads(args) else _alone(): + return compose(a.stack, args) + if a.op == "configure": + path, settings = configure(a.stack, a.suffix, a.write_mbps, a.write_iops) + out = {"settings": path, **settings, "sum_of_limits": dict(zip(("bps", "iops"), limits(settings)[1]))} + elif a.op == "builder": + out = builder(a.stack, a.op_, a.write_mbps, a.write_iops) + elif a.op == "build": + out = build(a.stack, a.group, a.reserve_gb) + elif a.op == "check": + if not SUFFIX.fullmatch(a.suffix): + raise WebsiteError(f"{a.suffix!r} is not a host suffix") + out = {"suffix": a.suffix, "names": ready(a.suffix, overleaf=True)} + elif a.op == "smoke": + out = smoke(a.stack) + else: + out = {"names": reset(a.stack)} + except (WebsiteError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/nullius/providers.py b/app/nullius/providers.py index 22c65bf..bf1d2da 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -16,6 +16,7 @@ import http.client import json import os +import re import secrets import socket import ssl @@ -211,6 +212,53 @@ def list_models(provider: str, env=None, *, opener=None) -> list[dict]: raise ValueError("Model list unavailable. Enter a model ID manually.") from None +# A model OpenRouter serves: its maker, a slash, its name (a variant after a colon: ``:free``). +_ROUTED = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._:-]*") +# A serving provider's slug, as ``--served-by`` takes it (``standard.relay.SLUG``). +_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def _endpoints(content: bytes) -> list[dict]: + """The serving providers of OpenRouter's endpoint list (``{"data": {"endpoints": [...]}}``), each + ``{slug, name, quantization}``: the slug its ``tag`` (a provider and its variant, ``deepinfra/fp8``), + else the provider's name lowercased with its spaces as dashes. ValueError when there is no list.""" + data = json.loads(content) + rows = ((data.get("data") or {}).get("endpoints") if isinstance(data, dict) and isinstance(data.get("data"), dict) else None) + if not isinstance(rows, list): + raise ValueError("missing endpoint list") + out = {} + for row in rows: + if not isinstance(row, dict): + continue + name = row.get("provider_name") if isinstance(row.get("provider_name"), str) else "" + tag = row.get("tag") if isinstance(row.get("tag"), str) else "" + slug = tag or re.sub(r"\s+", "-", name.strip().lower()) + if not slug or len(slug) > 128 or not _SLUG.fullmatch(slug) or slug in out: + continue + quant = row.get("quantization") if isinstance(row.get("quantization"), str) else "" + out[slug] = {"slug": slug, "name": (name or slug)[:128], "quantization": quant[:32]} + return sorted(out.values(), key=lambda e: (e["name"].casefold(), e["slug"])) + + +def list_endpoints(provider: str, model: str, env=None, *, opener=None) -> list[dict]: + """The serving providers OpenRouter lists for ``model`` (``GET /models/{author}/{slug}/endpoints``), for the + builder's Served by: asked as the model list is (its key when set, no redirect, nothing of the error echoed).""" + env = os.environ if env is None else env + provider, endpoint = split(provider) + if provider != "openrouter": + raise ValueError("Only OpenRouter routes a model to several providers.") + model = served_as(str(model or "").strip(), provider) + if not _ROUTED.fullmatch(model): + raise ValueError("Choose a model as OpenRouter names it, its maker first: maker/model.") + base, key = _where(provider, endpoint, env) + author, _, name = model.partition("/") + url = f"{base}/models/{urllib.parse.quote(author, safe='')}/{urllib.parse.quote(name, safe=':')}/endpoints" + try: + return _endpoints(_get(url, _bearer(key), opener)) + except (OSError, ValueError, TypeError, AttributeError, http.client.HTTPException): + raise ValueError("Provider list unavailable. Enter a provider's slug by hand.") from None + + # Settings' Test, per provider: only requests that cost nothing. A model list everywhere; OpenRouter's is # public, so its key is asked at its own key-information endpoint. TypeSafe has no free request: no Test. TESTED = ("openrouter", "openai", "anthropic", "custom") diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 56be6dd..2cd0065 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,9 +7,9 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] - PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] + PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force] PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed @@ -43,6 +43,9 @@ puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option`` sets one of the agent's or capability's options (``options`` in its definition); the rest keep the vendor's defaults, and the job's config records them all. +``--served-by [MODEL=]SLUG[,SLUG...]`` lets only those serving providers answer +a model on OpenRouter, in that order (``run.served_by``); each call's record +names the provider that served it. A run says each stage before its first unit, and each unit's end, as one ``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl`` (``nullius/progress.py``): what the run's page shows while it goes. @@ -525,6 +528,7 @@ def cmd_run(a): licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), + **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) @@ -596,11 +600,16 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) + jobs = options(a.job) + for c in jobs: + if c not in definitions.CONTROLS: + sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}") held, spill = licences(b) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, - env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images) + env=run_env(a.provider) if a.from_run else None, licences=held, + bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images) except run.Refused as e: print("refused:", e) return 2 @@ -694,6 +703,9 @@ def main(argv=None): p.add_argument("--constraint", action="append", metavar="KEY=VALUE", help="what the machine allows the run, repeatable: cpus, memory_mb, processes and cpu_set on its " "containers and, through a systemd scope, on its processes here; concurrency; gpu_ids (empty for none)") + p.add_argument("--served-by", action="append", metavar="[MODEL=]SLUG[,SLUG...]", + help="on OpenRouter, the serving providers that may answer a model, tried in this order and no " + "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") @@ -707,6 +719,10 @@ def main(argv=None): v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)") v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only") v.add_argument("--launch", help="the launch id the control jobs of every shard record") + v.add_argument("--bench-option", action="append", metavar="NAME=VALUE", + help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs") + v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR", + help="hold this job, a control run already over the whole dataset (a merged launch), to its bound instead of running it") v.add_argument("--remove-images", action="store_true", help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)") g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bafb447..9153d21 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + timeout: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -732,7 +733,9 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi an empty folder). ``hosts``: names it resolves (``--add-host``), the machine's own for an endpoint on its loopback. ``ledger``: the file of the run's spending cap, its folder mounted so - this relay books its calls where every other relay of the job does.""" + this relay books its calls where every other relay of the job does. + ``serving``: by model, the serving providers that may answer it, as the + unit's own relay is given them (``relay.Relay``).""" a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init", *capped(), *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")], @@ -752,8 +755,15 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] + # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's + sends = {model} | set(routes.values()) | {str(s).partition("=")[2] or str(s) for s in side_models} + for m, slugs in sorted((serving or {}).items()): + if m in sends: + a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] + if timeout: + a += ["--timeout", str(timeout)] return a @@ -1014,9 +1024,12 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True): + ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, + serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one + self.serving = dict(serving or {}) # by model, the serving providers that may answer it + self.timeout = timeout self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1042,7 +1055,8 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, + timeout=self.timeout) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 125c2b2..20b4f1f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -81,7 +81,7 @@ import re from typing import Callable, Dict, List, Optional, Tuple -from nullius.standard import ports +from nullius.standard import ports, relay from nullius import paths from nullius.catalogue import storage from . import names @@ -1035,6 +1035,25 @@ def control(bench: dict, which: str) -> Optional[dict]: "_kind": "control", "_ref": which, "_here": bench.get("_here", "")} +def comparable(orig: dict, model: str) -> bool: + """Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do.""" + c = orig.get("comparable", True) + return model in c if isinstance(c, list) else bool(c) + + +def upstream_name(orig: dict, model: str, provider: str) -> str: + """``model`` by the name the original's upstream knows, when given as ``provider`` lists it.""" + pre = (orig.get("providers") or {}).get(provider, "") + names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []} + return model if model in (orig.get("models") or []) else names.get(model, model) + + +def paper_models(orig: dict) -> List[str]: + """The original's models whose runs compare with the paper's.""" + c = orig.get("comparable", True) + return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c] + + def _leaderboard(d: dict) -> List[str]: """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the upstream's leaderboard fixes besides the ranked model, from its own submission @@ -1066,12 +1085,13 @@ def _leaderboard(d: dict) -> List[str]: def _setups(s) -> List[str]: """``setups``: {original: {summary, models, default, served_as?, providers, - side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s + side_models?, pin?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s ``models`` are the paper's, by the names the upstream's code knows them; ``side_models`` the ones its code asks besides the answerer (an extraction, a summary), sent as asked through served_as and the provider's prefix; ``providers`` the ones that serve them, each with the prefix its names - carry there; ``model`` needs ``original`` (its pipeline) and asks as one + carry there; ``pin`` {openrouter: {model: [provider slug]}}, the only + providers of each; ``model`` needs ``original`` (its pipeline) and asks as one of its models (``runs_as``).""" if s is None: return [] @@ -1090,7 +1110,7 @@ def _setups(s) -> List[str]: bad.append(f"{w}.summary: what runs, in a sentence") allowed = {"summary"} if k == "original": - allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"} + allowed |= {"models", "default", "served_as", "providers", "side_models", "pin", "units", "turn_seconds", "paper", "comparable", "reference_agent"} if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()): bad.append(f"{w}.reference_agent: the upstream agent's display name") models = x.get("models") @@ -1114,6 +1134,8 @@ def _setups(s) -> List[str]: pv = x.get("providers") if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()): bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}") + if "pin" in x: + bad += _pin(x["pin"], f"{w}.pin", pv if isinstance(pv, dict) else {}, set(models + side)) u = x.get("units") if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))): bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is") @@ -1122,8 +1144,11 @@ def _setups(s) -> List[str]: bad.append(f"{w}.turn_seconds: a whole number of seconds") if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()): bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase") - if "comparable" in x and not isinstance(x["comparable"], bool): - bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only") + c = x.get("comparable", True) + if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c) + or x.get("default") not in c): + bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; " + "or those of its models whose runs compare with the paper's, its default among them") else: allowed |= {"runs_as"} orig = s.get("original") if isinstance(s.get("original"), dict) else None @@ -1139,6 +1164,28 @@ def _setups(s) -> List[str]: return bad +PIN_PROVIDERS = ("openrouter",) + + +def _pin(pin, w: str, providers: dict, models: Optional[set] = None) -> List[str]: + """``pin``: {provider: {model: [slug]}} for a setup (``models`` given), {provider: [slug]} for a judge.""" + if not isinstance(pin, dict) or not pin: + return [f"{w}: {{openrouter: ...}}, the providers that alone serve it"] + bad = [] + for p, v in pin.items(): + if p not in PIN_PROVIDERS or (providers and p not in providers): + bad.append(f"{w}: a pin is OpenRouter's provider routing, on a provider the benchmark runs on ({p!r})") + continue + pins = v if models is not None else {"_": v} + if not isinstance(pins, dict) or not pins: + bad.append(f"{w}.{p}: {{model: [provider slug, ...]}}") + continue + if models is not None and not all(m in models for m in pins): + bad.append(f"{w}.{p}: pins only its models and side_models") + bad += [f"{w}.{p}: " + b.removeprefix("_: ") for b in relay.serving_problems(pins)] + return bad + + # Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder). SETTINGS_FILE = "upstream-settings.json" _DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$") @@ -1204,7 +1251,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple return None, f"{bench['label']} has no {which} setup" orig, x = s["original"], s[which] model = (model or "").strip() - side = [] + side, pins = [], {} if which == "original" and "models" not in orig: # any model, sent as the provider lists it if not model: return None, f"the original {bench['label']} needs a model: its leaderboard ranks models" @@ -1215,7 +1262,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple model = model or orig["default"] if model not in orig["models"]: return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): " - f"{model} is not one - choose the model setup for another") + f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else "")) if provider and provider not in orig["providers"]: return None, (f"the paper's models are not served by {provider}: the original runs on " + " or ".join(orig["providers"])) @@ -1225,13 +1272,15 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple sent = served(model) # the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent] + pins = {served(m): list(only) for m, only in ((orig.get("pin") or {}).get(provider) or {}).items() + if m == model or m in (orig.get("side_models") or [])} else: if not model: return None, f"the model setup needs a model: the one to put in {orig['default']}'s place" asked, sent = x["runs_as"], model units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"} - return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", - "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True), + return {"model": {"side_models": side}, **({"pins": pins} if pins else {}), "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", + "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model), "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which, "_here": bench.get("_here", "")}, "" @@ -1313,14 +1362,18 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") + if lim.get("run_again") is not None and not isinstance(lim["run_again"], bool): + bad.append("limits.run_again: false when a rate-limited unit is not run once more") a = lim.get("attempts") if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " "in a sentence - the driver is then sent attempt with setup, turn and answer") + if "gated" in d and not isinstance(d["gated"], bool): + bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept") s = d.get("score") or {} if s.get("kind") not in SCORES or not s.get("says"): bad.append(f"score: {{kind: one of {SCORES}, says}}") @@ -1332,6 +1385,8 @@ def problems(d: dict, kind: str) -> List[str]: j = s.get("judge") if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")): bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}") + elif j is not None and "pin" in j: + bad += _pin(j["pin"], "score.judge.pin", {}) sim = d.get("simulator") if sim is not None: if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"): diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index bc60de2..cd79086 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,8 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True).start() + key_header=run.PROVIDERS[provider].get("key_header", "authorization"), fields=True, + timeout=r.call_seconds).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} @@ -230,7 +231,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k except (run._Ended, jsonl.Failed) as e: r.say(f" {tag} {unit}: {e}") try: - score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600) + score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds) except jsonl.Failed: score = {} if os.path.isdir(os.path.join(base, "agent")): diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 683d4e1..a4cd73c 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -57,9 +57,9 @@ def _text(content) -> str: def _usage_total(calls: List[dict]) -> dict: - ok = [c for c in calls if c.get("status") == 200] + ok = [c for c in calls if c.get("status") == 200 or c.get("reply")] u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok) - costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))] + costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float)) and c.get("cost_from") != "failed"] return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"), "cost": sum(costs) if costs else None, "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None), @@ -169,6 +169,9 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili reply = c.get("reply") or {} st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"), "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"} + if c.get("status") == 200 and reply.get("tool_calls"): + st["tool_calls"] = [{"tool_call_id": f"call-{len(steps) + 1}-{k + 1}", "function_name": t.get("name") or "", + "arguments": _arguments(t.get("arguments"))} for k, t in enumerate(reply["tool_calls"])] if reply.get("usage"): st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"], "cached_tokens": reply["usage"]["cached"]} @@ -183,6 +186,17 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}} +def _arguments(said) -> dict: + """A tool call's arguments as a dict; text that is no JSON object goes under ``raw``.""" + try: + got = json.loads(said) if isinstance(said, str) else said + except ValueError: + got = None + if said is None: + return {} + return got if isinstance(got, dict) else {"raw": said if isinstance(said, str) else json.dumps(said)} + + def _control(other: dict) -> dict: """A control's run says so, and who wrote the control: the runs pages read a standard job named ``nop`` or ``oracle`` as a control only by this.""" @@ -317,7 +331,7 @@ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tu if at is not None: turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True, content={"input": st.get("message")})) - elif agent or st.get("tool_calls"): + elif agent or (st.get("tool_calls") and not st.get("model_name")): spans.extend(_step_spans(i + 1, st, source)) if not spans and not turns: return None @@ -400,11 +414,74 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: return "full" if full else "partial" +def _named(name: str) -> str: + """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``DeepInfra`` reads + ``deepinfra``) and nothing else folded: ``Prov/A`` is not ``prov-a``, and a provider whose name differs from its + slug otherwise (``Google`` serving ``google-vertex``) reads as another. A run fetches no list of + providers, and this is all a name is matched by.""" + return re.sub(r"\s+", "-", name.strip().lower()) + + +def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> Optional[dict]: + """Who served a unit's answered calls (each record's ``reply.served_by``): the calls by serving provider and + those that named none; and, for each model the run pinned (``serving``), whether the pin held - true when + every answered call sent as that model names a provider that reads as one of its pinned slugs (``_named``, + a slug's first part), ``unknown`` when one names none or another name, None when it has no answered call. + A slug's endpoint variant (``/fp8``) is not in a reply's name: ``held`` says which provider answered, and + cannot tell one of its endpoints from another. None for a unit with no answered call in a run with no pin.""" + answered = [c for c in calls if c.get("status") == 200 and not (c.get("reply") or {}).get("error")] + by: Dict[str, int] = {} + unnamed = 0 + for c in answered: + name = (c.get("reply") or {}).get("served_by") + if isinstance(name, str) and name: + by[name] = by.get(name, 0) + 1 + else: + unnamed += 1 + if not answered and not serving: + return None + out = {"calls": dict(sorted(by.items())), "unnamed": unnamed} + if serving: + held = {} + for m, slugs in sorted(serving.items()): + mine = [c for c in answered if c.get("model") == m] + pinned = {str(s).split("/")[0].lower() for s in slugs} + held[m] = None if not mine else True if all( + _named(str((c.get("reply") or {}).get("served_by") or "")) in pinned for c in mine) else "unknown" + out["held"] = held + return out + + +def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[dict]: + """The job's ``served``: its trials' added up with ``also`` (the benchmark's own relay's, :func:`served`), a + pin held where it held in every one that called its model.""" + parts = [p for p in [(r.get("standard") or {}).get("served") for r in results] + [also] if p] + if not parts: + return None + calls: Dict[str, int] = {} + for p in parts: + for name, n in (p.get("calls") or {}).items(): + calls[name] = calls.get(name, 0) + n + out = {"calls": dict(sorted(calls.items())), "unnamed": sum(p.get("unnamed") or 0 for p in parts)} + models = sorted({m for p in parts for m in (p.get("held") or {})}) + if models: + out["held"] = {} + for m in models: + said = [p["held"][m] for p in parts if (p.get("held") or {}).get(m) is not None] + out["held"][m] = None if not said else True if all(v is True for v in said) else "unknown" + return out + + def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict: - """Write one unit's trial and return its result. ``environment``: the + image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), + serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None) -> dict: + """Write one unit's trial and return its result. ``serving``: the run's + pin of serving providers by model; ``standard.served`` says who served + ``served_calls`` (the unit's calls on every route of its relay: its + agent's, its judge's, its simulated user's; ``calls`` when not given) and + whether the pin held (:func:`served`). ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began @@ -426,13 +503,16 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]}, "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), - **({"environment": environment} if environment else {}), **({"image": image} if image else {})}} + **({"environment": environment} if environment else {}), **({"image": image} if image else {}), + **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score # benchmark a score and no pass or fail - the trial reads as Scored rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value} res["verifier_result"] = {"rewards": rewards} + if score.get("stopped"): + res["verifier_result"]["stopped"] = str(score["stopped"]) parts = score.get("parts") if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0: # a score over several parts (a conversation's questions): the job's mean weighs it by them @@ -451,9 +531,15 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) if trajectory is not None: _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1)) - timeline = trace_from_run(trajectory, calls, other, produce) - if timeline is not None: - _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + try: + timeline = trace_from_run(trajectory, calls, other, produce) + if timeline is not None: + _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + except Exception as e: + try: + _write(os.path.join(trial_dir, "agent", "trace.error.txt"), type(e).__name__ + "\n") + except OSError: + pass for name, text in (extra_logs or {}).items(): _write(os.path.join(trial_dir, "agent", name), text) _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1)) @@ -465,7 +551,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -477,7 +564,9 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str ``simulator``: the models that graded and that played the user, when the benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. - ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any.""" + ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any. + ``served_by``: by model, the serving providers the run let answer it + (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -488,7 +577,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), - **({"constraints": constraints} if constraints else {})}, + **({"constraints": constraints} if constraints else {}), + **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], @@ -544,14 +634,18 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} -def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): +def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None, + served: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, what was spent, whether it stopped the run, the units it cut off and - those never begun.""" + those never begun. ``served``: who served the benchmark's own relay's + calls (:func:`served`); the job's ``served`` adds its trials' to them.""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) + who = served_total(results, served) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, - **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1) + **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {}), + **({"served": who} if who else {})}, f, indent=1) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index c0d6636..955df60 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -9,8 +9,9 @@ the shards' with ``shard`` taken out and ``merged_from`` put in. What is merged must be one launch: the same launch id, benchmark and -version, dataset, agent or capability and model, and the same number of shards -- else it is refused, naming what differs, unless ``force``. Two trials of +version, dataset, agent or capability and model, the same serving providers +pinned for each model (``served_by``) and the same number of shards - else it +is refused, naming what differs, unless ``force``. Two trials of one unit are always refused, with the units named: one of them would be counted twice. So are shards whose lists of units (``run.units_file``) hold different units, or a shard of a list and one of none: the merged job @@ -18,7 +19,11 @@ comments, line endings and sha256 may differ from shard to shard, and the merged job records the first shard's list. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: -the shards absent, and the units a shard planned and has no trial for. +the shards absent, and the units a shard planned and has no trial for. A +trial whose record names another job, benchmark, version, control or agent +than its shard, or a job itself merged despite differences, is refused +unless ``force``. Shards run with different benchmark options keep them each +in ``merged_from``. Files are hard-linked into the merged job where the file system allows (a copy where it does not), so a merge of a thousand trials costs no second @@ -40,7 +45,13 @@ SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")), ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")), - ("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of"))) + ("backend", ("standard", "backend")), ("serving providers", ("standard", "served_by")), + ("number of shards", ("standard", "shard", "of"))) + + +LISTED = 1000 +OWN = (("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), + ("control", ("standard", "control", "name")), ("agent or capability", ("standard", "plugged"))) class Refused(Exception): @@ -73,6 +84,28 @@ def _listed(cfg: dict) -> Optional[tuple]: return tuple(sorted(str(u) for u in units)) if isinstance(units, list) else None +def _served(shards: List[dict], results: List[dict]) -> Optional[dict]: + """The merged job's ``served`` (``job.served_total``), made of each shard's whole one - its trials' and its + benchmark relay's calls, an ``unknown`` kept - else, for a shard that has none, of its trials'. A model the + shards pinned differently - merged with ``force`` - reads ``unknown``: the merged config keeps the first + shard's pin, which the others' calls were never held to.""" + parts = [] + for s in shards: + whole = (s.get("result") or {}).get("served") + if isinstance(whole, dict): + parts.append(whole) + else: + parts.append(job.served_total([r for r in results if (r.get("standard") or {}).get("shard_job") == s.get("name")])) + who = job.served_total([{"standard": {"served": p}} for p in parts if p]) + pins = [_at(s["config"], ("standard", "served_by")) or {} for s in shards] + pins = [p if isinstance(p, dict) else {} for p in pins] + differ = sorted(m for m in {m for p in pins for m in p} if len({json.dumps(p.get(m)) for p in pins}) > 1) + if differ: + who = who or {"calls": {}, "unnamed": 0} + who["held"] = dict(who.get("held") or {}, **{m: "unknown" for m in differ}) + return who + + def _trials(job_dir: str) -> List[str]: return sorted(n for n in os.listdir(job_dir) if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json"))) @@ -123,7 +156,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: + " (--force does not merge them)") differ = [] for label, path in SAME: - seen = {json.dumps(_at(s["config"], path)) for s in shards} + seen = {json.dumps(_at(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content if len(seen) > 1: differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards)) if differ and not force: @@ -131,12 +164,30 @@ def check(job_dirs: List[str], force: bool = False) -> dict: # every trial by its unit: two of one unit are never merged, force or not by_unit: Dict[str, List[tuple]] = {} names: Dict[str, List[str]] = {} + foreign = [] for s in shards: s["trials"] = _trials(s["dir"]) for t in s["trials"]: - unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t + res = _read(os.path.join(s["dir"], t, "result.json")) + unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) + wrong = [label for label, path in OWN if _at(res, path) != _at(s["config"], path)] + if _at(res, ("standard", "job")) != (s["config"].get("job_name") or s["name"]): + wrong.append("job") + if wrong: + foreign.append(f"{s['name']}/{t} names another {', '.join(wrong)} than its job") + inherited = [f"{s['name']}: {w}" for s in shards for w in _at(s["config"], ("standard", "merged_despite")) or []] + if inherited: + if not force: + raise Refused("a job given was itself merged despite differences - " + "; ".join(inherited[:5]) + + " (--force merges it anyway, and the merge says so)") + differ += inherited + if foreign: + shown = f"{len(foreign)} trial(s) not of the job they are in: " + "; ".join(foreign[:5]) + (" ..." if len(foreign) > 5 else "") + if not force: + raise Refused(shown + " (--force merges them anyway, and the merge says so)") + differ.append(shown) twice = {u: w for u, w in by_unit.items() if len(w) > 1} if twice: shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10]) @@ -151,9 +202,10 @@ def check(job_dirs: List[str], force: bool = False) -> dict: if len(of) == 1: n = of.pop() have = sorted(s["shard"].get("index") for s in shards if s["shard"]) - absent = [i for i in range(1, (n or 0) + 1) if i not in have] - if absent: - notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") + if isinstance(n, int) and n <= LISTED: + absent = [i for i in range(1, n + 1) if i not in have] + if absent: + notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") dup = sorted({i for i in have if have.count(i) > 1}) if dup: notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)") @@ -241,13 +293,21 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), "scored": len(scores), "mean_score": job.mean_score(results)}, - **({"budget": budget} if budget else {})}, f, indent=1) + **({"budget": budget} if budget else {}), + **({"served": who} if (who := _served(shards, results)) else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) std.pop("shard", None) + own = len({json.dumps(_at(s["config"], ("standard", "benchmark_options")), sort_keys=True) for s in shards}) > 1 + if own: + std.pop("benchmark_options", None) std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]), - **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards] + **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {}), + **({"benchmark_options": _at(s["config"], ("standard", "benchmark_options")) or {}} if own else {}), + **({"merged_from": _at(s["config"], ("standard", "merged_from"))} + if _at(s["config"], ("standard", "merged_from")) else {})} + for s in shards] if got["units_total"] is not None: std["units_total"] = got["units_total"] if got["differ"]: diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318..8b39142 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -30,6 +30,8 @@ pages pair them as a launch's controls); the units and leak checks talk to the driver directly and leave nothing behind. A control calls no model, so it needs no key - except a judged benchmark's, whose judge still grades. +``--bench-option`` reaches the checks' driver and each control's run; +``--job nop=DIR`` holds a merged launch of a control to its bound. A check that passes is recorded with its date, its counts and its job; one that fails is taken back, as ``conform`` takes back a port - what it reached kept @@ -41,6 +43,8 @@ """ from __future__ import annotations +import json +import math import os import re import shutil @@ -48,8 +52,7 @@ import time from typing import Callable, Dict, List, Optional -from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, run, shard as shards +from nullius.standard import definitions, install, job, jsonl, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -66,7 +69,8 @@ class _Driver: """The benchmark's driver, started as a run starts it, for the checks that talk to it directly (units, leak).""" - def __init__(self, bench: dict, ds: dict, work: str, licences=None): + def __init__(self, bench: dict, ds: dict, work: str, licences=None, options: Optional[dict] = None): + self.options = options if options is not None else options_for(bench, ds) self.up = install.upstream(bench) self.data = install.dataset(bench, ds, self.up) install.driver_packages(bench) # its driver.install, once: start() then finds it @@ -81,10 +85,15 @@ def start(self, n: int) -> jsonl.Proc: env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), - **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults + **run.option_env(self.options)}) return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() +def options_for(bench: dict, ds: dict, given: Optional[dict] = None, which=shutil.which) -> dict: + """The benchmark's options on ``ds`` for a control's run, at ``given`` or their defaults.""" + return run.resolve_bench_options(bench, ds["id"], given, "yours", which)[0] + + def units_check(driver: _Driver, ds: dict) -> dict: """Two drivers, one ``units`` call each: the same ids, unique, plain.""" got = [] @@ -209,44 +218,164 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str, provider: str, model: str, judge_model: Optional[str], env, concurrency: int, say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None, - licences=None, remove_images: bool = False) -> dict: + licences=None, options: Optional[dict] = None, job_dir: Optional[str] = None, + ids: Optional[List[str]] = None, remove_images: bool = False) -> dict: """Run one control over the dataset (or its first ``limit`` units, of - ``shard`` when given) and hold it to its bound: the pass rate, or the mean score.""" + ``shard`` when given) and hold it to its bound: the pass rate, or the mean score. + ``job_dir``: a control run already, held to the bound instead; it must cover ``ids``.""" other = definitions.control(bench, which) - job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, - jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), - env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images) - j = parse.parse_job(job_dir) - n = len(j.trials) - errored = sum(1 for t in j.trials if t.exception_type) + problems = [] + if job_dir: + job_dir = os.path.abspath(job_dir.rstrip("/")) + problems += job_problems(bench, ds, which, job_dir, ids or []) + cfg = _config(job_dir) + else: + job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, + jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), + say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, + licences=licences, bench_options=options, + remove_images=remove_images) + cfg = {} + field = FIELD[bench["score"]["kind"]] + results = [res for _, res in trials(job_dir)] + n = len(results) + errored = sum(1 for res in results if res.get("exception_info")) + values = [verdict(bench, res) for res in results if not res.get("exception_info")] + unscored = sum(1 for v in values if v is None) + values = [v for v in values if v is not None] + top = sum(1 for v in values if v >= 1.0) if bench["score"]["kind"] == "pass": - top = sum(1 for t in j.trials if t.reward == 1.0) value = top / n if n else 0.0 else: - scores = [t.score for t in j.trials if t.score is not None] - top = sum(1 for s in scores if s >= 1.0) - value = sum(scores) / len(scores) if scores else 0.0 + value = sum(values) / len(values) if values else 0.0 (bound, expected), = other["expect"].items() within = value <= expected if bound == "at_most" else value >= expected - problems = [] if not n: problems.append("no unit ran") if errored: problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials") + if unscored: + problems.append(f"{unscored} unit(s) ended with no {field} from 0 to 1 and no error: read the job's trials") if n and not within: problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}") rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected}, "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems} - if judge_model or (bench["score"].get("judge") or {}).get("model"): - rec["judge"] = judge_model or bench["score"]["judge"]["model"] + ran_with = _options_of(cfg) if cfg else (options or {}) + if ran_with: + rec["options"] = ran_with + judge = (cfg.get("standard") or {}).get("judge") if cfg else judge_model or (bench["score"].get("judge") or {}).get("model") + if judge: + rec["judge"] = judge return rec +FIELD = {"pass": "reward", "score": "score"} +UNITS_FAILED = "the units check failed" + + +def trials(job_dir: str) -> List[tuple]: + """(trial, its result.json) for each trial folder of a job; an unreadable record is {}.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if name.startswith(".") or not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + res = json.load(f) + except (OSError, ValueError): + res = {} + out.append((name, res if isinstance(res, dict) else {})) + return out + + +def verdict(bench: dict, res: dict) -> Optional[float]: + """The trial's value in the field its benchmark's score kind names, when it is a finite number from 0 to 1.""" + v = (((res.get("verifier_result") or {}).get("rewards")) or {}).get(FIELD[bench["score"]["kind"]]) + if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0.0 <= v <= 1.0: + return None + return float(v) + + +def _options_of(cfg: dict): + """The benchmark options a job ran with: one set, or each distinct set of its merged shards.""" + std = cfg.get("standard") or {} + if std.get("benchmark_options"): + return std["benchmark_options"] + sets, todo = [], list(std.get("merged_from") or []) + while todo: + m = todo.pop() + if m.get("benchmark_options") or not m.get("merged_from"): + sets.append(m.get("benchmark_options") or {}) + else: + todo += m.get("merged_from") or [] + distinct = {json.dumps(x, sort_keys=True): x for x in sets} + return [distinct[k] for k in sorted(distinct)] if any(sets) else [] + + +def _config(job_dir: str) -> dict: + try: + with open(os.path.join(job_dir, "config.json"), encoding="utf-8") as f: + got = json.load(f) + return got if isinstance(got, dict) else {} + except (OSError, ValueError): + return {} + + +def job_problems(bench: dict, ds: dict, which: str, job_dir: str, ids: List[str]) -> List[str]: + """Why ``job_dir`` is not one run of ``which`` over each unit of ``ids`` once.""" + cfg = _config(job_dir) + std = cfg.get("standard") or {} + if cfg.get("runner") != "standard": + return [f"{job_dir}: not a job of the Nullius standard"] + want = {"benchmark": bench["key"], "benchmark version": job._version(bench).get("benchmark_version"), + "control": which, "dataset": f"{bench['key']}/{ds['id']}"} + have = {"benchmark": std.get("benchmark"), "benchmark version": std.get("benchmark_version"), + "control": (std.get("control") or {}).get("name"), + "dataset": ((cfg.get("datasets") or [{}])[0] or {}).get("name")} + bad = [f"its {k} is {have[k]!r}, not {want[k]!r}" for k in want if have[k] != want[k]] + if std.get("merged_despite"): + bad.append("merged despite differences: " + "; ".join(std["merged_despite"])) + if (std.get("repeat") or 1) > 1: + bad.append(f"each unit ran {std['repeat']} times") + ran: Dict[str, int] = {} + shard_jobs = {m.get("job") for m in std.get("merged_from") or []} + alien: Dict[str, List[str]] = {} + for trial, res in trials(job_dir): + tstd = res.get("standard") if isinstance(res.get("standard"), dict) else {} + own = {"benchmark": tstd.get("benchmark"), "benchmark version": tstd.get("benchmark_version"), + "control": (tstd.get("control") or {}).get("name"), + "who ran it": (tstd.get("plugged"), (res.get("agent_info") or {}).get("name")), + "job": tstd.get("job")} + expect = {"benchmark": want["benchmark"], "benchmark version": want["benchmark version"], "control": which, + "who ran it": (which, which), "job": cfg.get("job_name")} + wrong = [k for k in expect if own[k] != expect[k]] + if shard_jobs and tstd.get("shard_job") not in shard_jobs: + wrong.append("shard") + for k in wrong: + alien.setdefault(k, []).append(trial) + u = str(tstd.get("unit") or trial) + ran[u] = ran.get(u, 0) + 1 + for k, names in alien.items(): + bad.append(f"{len(names)} trial(s) whose own record names another {k} than the job: {', '.join(names[:3])}") + known = set(ids) + missing = [u for u in ids if u not in ran] + extra = sorted(u for u in ran if u not in known) + twice = sorted(u for u, k in ran.items() if k > 1) + if missing: + bad.append(f"{len(missing)} of the dataset's {len(ids)} units have no trial: {', '.join(missing[:5])}" + (" ..." if len(missing) > 5 else "")) + if extra: + bad.append(f"{len(extra)} trial(s) of units the dataset does not have: {', '.join(extra[:5])}") + if twice: + bad.append(f"{len(twice)} unit(s) with more than one trial: {', '.join(twice[:5])}") + return [f"{os.path.basename(job_dir)}: {b}" for b in bad] + + def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs", provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None, env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print, shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None, + bench_options: Optional[dict] = None, jobs: Optional[Dict[str, str]] = None, remove_images: bool = False) -> Dict[str, dict]: """Every check on every dataset (or ``dataset``); with ``write``, what passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the @@ -254,13 +383,26 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = the units check all of them, and each record says which shard. ``remove_images``: each unit's own image is removed after it, so the oracle pulls it again (by default they stay). Returns - {dataset: {check: record}}, each record with ``ok`` and its ``problems``.""" + {dataset: {check: record}}, each record with ``ok`` and its ``problems``. + ``jobs``: {control: job directory}, held to its bound instead of run.""" env = env if env is not None else os.environ stamp = stamp or time.strftime(STAMP) + jobs = dict(jobs or {}) sets = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not sets: raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})] + if jobs: + if dataset is None and len(sets) > 1: + raise run.Refused("a control's job is a run on one dataset: name it (--dataset)") + if limit or (shard and shard[1] > 1): + raise run.Refused("a control's job is held over the whole dataset: no limit, no shard") + for c, d in jobs.items(): + if c not in declared: + raise run.Refused(f"{bench['label']} declares no {c} control") + if not os.path.isdir(d): + raise run.Refused(f"{d}: no such job") + opts = {ds["id"]: options_for(bench, ds, bench_options) for ds in sets} if not declared: say(f"{bench['label']} declares no controls: only its units and nothing else can be checked") got: Dict[str, dict] = {} @@ -270,11 +412,11 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = checks: Dict[str, dict] = {} try: try: - driver = _Driver(bench, ds, work, licences=licences) + driver = _Driver(bench, ds, work, licences=licences, options=opts[ds["id"]]) except install.InstallFailed as e: raise run.Refused(f"could not install what the proof needs: {e}") checks["units"] = units_check(driver, ds) - ids = checks["units"].pop("ids") + every = ids = checks["units"].pop("ids") if shard and shard[1] > 1: ids = [u for _, u in shards.select(ids, *shard)] ids = ids[:limit] if limit else ids @@ -285,10 +427,18 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = finally: shutil.rmtree(work, ignore_errors=True) for c in declared: - checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, - model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, - shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, - remove_images=remove_images) + if not checks["units"]["ok"] and not jobs.get(c): + checks[c] = {"ok": False, "units": 0, "source": definitions.control(bench, c)["source"], + "problems": [UNITS_FAILED + ": the control was not run"]} + else: + checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, + model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, + shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, + options=opts[ds["id"]], job_dir=jobs.get(c), ids=every, + remove_images=remove_images) + if not checks["units"]["ok"]: + checks[c]["problems"].append(UNITS_FAILED + ": a job's cover is held to the driver's ids, which did not pass") + checks[c]["ok"] = False _say(say, c, checks[c]) total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"] for name, r in checks.items(): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 9e397d1..b6a6a73 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -40,7 +40,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove +from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay HARNESS = "standard" # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's. @@ -49,6 +49,8 @@ MAX_CONCURRENT = 32 BACKENDS = definitions.names.BACKENDS _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") +# A pin's field, as the Review's Model row names it (served_by, --served-by) +SERVED_LABEL = {"model": "model", "simulator": "user simulator", "judge": "judge"} _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$") @@ -124,9 +126,10 @@ def _bench_page(b: dict) -> dict: # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], "reference_agent": x.get("reference_agent", ""), - "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), - **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), - "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} + "comparable": k == "original" and b["setups"]["original"].get("comparable", True) is not False, "units": (b["setups"]["original"].get("units") or b.get("units") or {}), + **({"models": definitions.paper_models(x), "default": x["default"], "providers": list(x["providers"]), + "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in definitions.paper_models(x)} + for p, pre in x["providers"].items()}} if "models" in x else {"any": True, "providers": list(x["providers"])})} for k, x in (b.get("setups") or {}).items() if k == "original"], # its own options (a search, a mode): each value with its line and what it needs; and the host requirements @@ -281,11 +284,17 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it names have no price in the user's list (``nullius.prices``): a run - with a cap is refused without one, and the Limits step says so first.""" + with a cap is refused without one, and the Limits step says so first. + + ``served_by``: on OpenRouter, the serving providers each model field + pins, ``{model|simulator|judge: [slug, ...]}``, checked as ``--served-by`` + is and compiled into it for each job that calls that model, by the name + the run sends it as.""" d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) - out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""} + out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "", + "served_models": {}} name = str(recipe.get("name") or "") if not _NAME.match(name): add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.") @@ -331,6 +340,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "Choose a model.") continue method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" + if method == "original": + picked = definitions.upstream_name(orig, picked, provider_) o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") @@ -407,8 +418,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()): add("model", "No paper models are listed for this benchmark; choose Any of your choice.") elif model_choice == "paper" and needs_model and orig.get("models"): - if recipe.get("model") not in orig["models"]: - add("model", "Choose one of the paper's listed models.") + if recipe.get("model") not in definitions.paper_models(orig): + add("model", "Choose one of the paper's listed models, or Any of your choice for another of the original's.") elif provider_ not in orig.get("providers", {}): add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") @@ -437,6 +448,20 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model + # the serving providers each model field pins, on OpenRouter (served_by: {model|simulator|judge: [slug, ...]}), + # checked as --served-by is (relay.serving_problems); an empty list pins nothing + roles = {"model": "the model", "simulator": "the user simulator's model", "judge": "the judge's model"} + pins = recipe.get("served_by") + served: Dict[str, List[str]] = {} + if pins not in (None, {}): + if not isinstance(pins, dict) or any(k not in roles for k in pins): + add("model", "Served by names the serving providers of the model, the user simulator's or the judge's.") + else: + served = {k: v for k, v in pins.items() if v not in (None, [], "")} + for problem in relay.serving_problems({roles[k]: v for k, v in served.items()}): + add("model", "Served by " + problem + ".") + if served and provider_ != "openrouter": + add("model", "Served by needs OpenRouter, the provider that routes a model to several: choose it, or leave Served by empty.") # the run's spending cap: an amount in USD, or none cap = recipe.get("max_usd") if cap in ("", None): @@ -495,6 +520,42 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, limits.of(t) except ValueError as e: add("where", f"{e}: fix it in Settings.") + judge_def, sim_def = (b.get("score") or {}).get("judge"), b.get("simulator") + + def sent_of(p: dict) -> Dict[str, Optional[str]]: + """Each model field's model as job ``p`` sends it; None where it has none (a control has no model of its own).""" + return {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, + "judge": (judge_model or providers.served_as(judge_def["model"], provider)) if judge_def else None, + "simulator": (sim_model or providers.served_as(sim_def["model"], provider)) if sim_def else None} + + def called_of(p: dict) -> List[str]: + s = sent_of(p) + return ledger.models(p["o"], s["model"], s["judge"], s["simulator"], None, control=p["control"]) + + # the pins by model, as the run sends each: a field's pin is its model's, on every job that calls that model + # (a control's judge job too, when the judge's model is the one the model field pinned) + by_model: Dict[str, List[str]] = {} + for p in picks: + for role, slugs in served.items(): + m = sent_of(p).get(role) + if not m: + continue + if m in by_model and by_model[m] != list(slugs): + add("model", f"Served by pins {m} twice, with different providers: give its fields the same ones.") + by_model[m] = list(slugs) + # each field's model as the run sends it, for the builder to ask its providers under that name + out["served_models"] = {role: next((sent_of(p)[role] for p in sorted(picks, key=lambda q: q["control"]) if sent_of(p)[role]), None) + for role in roles} + + def pinned(p: dict) -> Dict[str, List[str]]: + """The --served-by of one job: the pins of the models it calls.""" + called = called_of(p) + return {m: s for m, s in by_model.items() if m in called} + if served and not errors: + everything = {m for p in picks for m in called_of(p)} + for role in served: + if not {sent_of(p)[role] for p in picks} & everything: + add("model", f"Served by: no job of this recipe calls {roles[role]}; leave it empty.") if errors: return out nouns = (b.get("units") or {}).get("nouns", "units") @@ -535,7 +596,8 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--backend", "container"] if backend == "container" else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch] if of > 1 else []), - *limits.cli_words(limits.of(targets[index - 1])), "--from-run"] + *limits.cli_words(limits.of(targets[index - 1])), + *[w for m_, slugs in pinned(p).items() for w in ("--served-by", f"{m_}={','.join(slugs)}")], "--from-run"] return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"' def script_for(d: dict, p: dict, command: str, index: int) -> str: @@ -615,7 +677,8 @@ def row(p: dict) -> list: [STEP["model"], "Model", "none: a control calls no model" if out["keyless"] else f"{providers.label(providers.choice(provider, str(recipe.get('endpoint') or '')))} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only") + (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "") - + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")], + + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + + "".join(f" · {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ") + (" of each dataset" if many and isinstance(limit, int) else "") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index a8c5945..76fa01d 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -9,10 +9,16 @@ byte for byte - and recorded once it is complete, one line per call in ``/.jsonl``: when it started and ended, the model, the status, the tokens, and an abridged request and reply. Three API shapes are read, -Chat Completions, Responses and Anthropic's Messages, streamed or not. The +Chat Completions, Responses and Anthropic's Messages, streamed or not. Read +apart from the rest (:func:`reply_facts`), the reply also says, when it does, +the serving provider that answered (``served_by``), how it ended (``end``) +and an error it holds though its status was a success (``error``); reading +them never changes what the caller receives, nor the rest of the record but a failed Responses object's cost. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the -user's price list, ``nullius.prices``), marked as an estimate. +user's price list, ``nullius.prices``), marked as an estimate. A Responses object answered 200 that failed (its ``end``) with no usage +costs 0, marked as failed: the provider keeps no generation for it, and a +key's usage compared with the relay's sum would show one billed after all. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -24,6 +30,12 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +A run may pin, for a model, the serving providers that may answer it +(``serving``, OpenRouter's routing): the relay then puts +``provider: {order, allow_fallbacks: false}`` on every request it sends as +that model, in place of the caller's own, and records both, cleaned of +credentials. Without a pin a request's body goes on as it did. + One answer is the relay's own: a run with a spending cap (``ledger``) books each call before it goes out, and a call that does not fit is not sent. Its caller gets a 402 that names the relay and the cap, and the record says @@ -31,6 +43,8 @@ """ from __future__ import annotations +import base64 +import binascii import hashlib import http.client import json @@ -67,6 +81,16 @@ # its next use, before any of the reply is read: worth one new connection. _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError) ANTHROPIC_VERSION = "2023-06-01" +ROUTE_REFUSED = (404, 503) +ROUTE_ERROR = "ApiProviderRouteError" +CONTEXT_ERROR = "ContextWindowExceededError" +STOP = "stop" +OVERFLOW = "context window" +COMPLETION = re.compile(r"^/(?:chat/completions|completions|messages|responses)$") +_WHOLE = ((b"\x89PNG\r\n\x1a\n", b"IEND\xaeB`\x82", "image/png"), (b"\xff\xd8\xff", b"\xff\xd9", "image/jpeg"), + (b"GIF87a", b";", "image/gif"), (b"GIF89a", b";", "image/gif")) +_PROVIDER = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._()/-]{0,79}") +_CONTEXT = re.compile(r"context[ _-]?(length|window|limit)|prompt is too long|token count .{0,40}exceeds", re.I) def _cut(text, n=4000): @@ -104,7 +128,7 @@ def _usage(u: Optional[dict]) -> Optional[dict]: inp += cached + written else: written = (det or {}).get("cache_write_tokens") or 0 - got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0} + got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or (rdet or {}).get("thinking_tokens") or 0} if written: got["cache_write"] = written hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0 @@ -138,6 +162,25 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def _hollow(path: str, body: bytes) -> Optional[tuple]: + """A 200 to a model call that holds no reply: the status it stands for, 429 or 402 when an error it held says so, else + 502, and its kind alone, since its words can quote the request.""" + if not COMPLETION.search(path.split("?")[0]): + return None + try: + got = json.loads(body) + except ValueError: + return 502, "no reply" + if not isinstance(got, dict): + return 502, "no reply" + if read_reply(path, body, False): + return None + code = got["error"].get("code") if isinstance(got.get("error"), dict) else None + code = code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code < 600 else None + return ((code if code in (402, 429) else 502), "an error" + (f" {code}" if code else "")) if got.get("error") \ + else (502, "no reply") + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: """What a reply said: its text, its tool calls, its usage.""" if not streamed: @@ -192,7 +235,7 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): t, c = _text_of_responses(ev.get("response") or {}) return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: @@ -211,6 +254,219 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: "usage": _usage(told) if told is not None else usage} +# -- serving providers: the pin a run sets, and what a reply says of who served it -- +SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def routing(slugs) -> dict: + """OpenRouter's routing field for a strict pin: these serving providers, + tried in this order, and no other.""" + return {"order": list(slugs), "allow_fallbacks": False} + + +def serving_problems(serving) -> List[str]: + """What is wrong with a pin by model (``{model: [slug, ...]}``): the one + check the run, its command line and the relay's own command share.""" + if not isinstance(serving, dict): + return ["the serving providers are given by model: {MODEL: [SLUG, ...]}"] + bad = [] + for model, slugs in serving.items(): + if not isinstance(model, str) or not model.strip(): + bad.append("a pin names no model") + continue + if not isinstance(slugs, (list, tuple)) or not slugs: + bad.append(f"{model}: no serving provider") + continue + wrong = [s for s in slugs if not isinstance(s, str) or not SLUG.fullmatch(s)] + if wrong: + bad.append(f"{model}: {', '.join(repr(s) for s in wrong)} is not a serving provider's slug " + "(as a model's page names it: deepinfra, deepinfra/fp8)") + elif len(set(slugs)) != len(slugs): + bad.append(f"{model}: a serving provider is named twice") + return bad + + +def parse_served_by(given, model: str) -> tuple: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable, as ``({model: [slug, ...]}, problems)``. Without + ``MODEL=`` it is ``model``, the run's. A model named twice is a problem; so is each of :func:`serving_problems`.""" + out: Dict[str, List[str]] = {} + bad = [] + for g in given or []: + name, eq, listed = str(g).rpartition("=") + name = name.strip() if eq else (model or "") + if not name: + bad.append(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") + continue + if name in out: + bad.append(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") + continue + out[name] = [s.strip() for s in listed.split(",")] + return out, bad + serving_problems(out) + + +def _clean(value, secrets=()): + """A recorded copy, never what is sent: a value under a credential's name + replaced, the relay's own key cut out of every string - a key of a + mapping as much as a value - then ``redact``.""" + if isinstance(value, dict): + return {_clean(k, secrets): "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) + for k, v in value.items()} + if isinstance(value, list): + return [_clean(v, secrets) for v in value] + if isinstance(value, str): + for s in secrets: + if s: + value = value.replace(s, "[REDACTED]") + return redact(value) + return value + + +def _name(v) -> Optional[str]: + return v if isinstance(v, str) and v else None + + +def _an_error(e) -> Optional[dict]: + """An error object, in one shape: its code (a word or a number), type and + message (words), as given; None when it has none of them of that kind. + Nothing else of the object is kept.""" + if not isinstance(e, dict) or not e: + return None + got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} + if isinstance(e.get("message"), str): + got["message"] = _cut(e["message"], 500) + return got or None + + +def _routing_facts(obj) -> Optional[list]: + """The serving endpoints OpenRouter's routing facts (``openrouter_metadata``) + list as available, in an object of a reply, its ``message`` or its + ``response``; None when none of them holds such a list.""" + if not isinstance(obj, dict): + return None + for o in (obj, obj.get("message"), obj.get("response")): + meta = o.get("openrouter_metadata") if isinstance(o, dict) else None + endpoints = meta.get("endpoints") if isinstance(meta, dict) else None + available = endpoints.get("available") if isinstance(endpoints, dict) else None + if isinstance(available, list): + return available + return None + + +def _provider(obj) -> Optional[str]: + """The serving provider an object of a reply names: its own ``provider``, + its ``message``'s or its ``response``'s, else the one endpoint OpenRouter's + routing facts mark ``selected`` among the available ones (asked for with + ``X-OpenRouter-Metadata``; a Responses reply names its provider there + alone). Facts that mark no single named endpoint give none.""" + if not isinstance(obj, dict): + return None + for o in [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)]: + if _name(o.get("provider")): + return o["provider"] + chosen = [a for a in _routing_facts(obj) or () if isinstance(a, dict) and a.get("selected") is True] + return _name(chosen[0].get("provider")) if len(chosen) == 1 else None + + +def _choices(choices, ends: dict, errors: dict) -> None: + """Each choice's end and error, by its index.""" + if not isinstance(choices, list): + return + for n, ch in enumerate(choices): + if not isinstance(ch, dict): + continue + i = ch.get("index") if isinstance(ch.get("index"), int) and not isinstance(ch.get("index"), bool) else n + if _name(ch.get("finish_reason")): + ends[i] = ch["finish_reason"] + e = _an_error(ch.get("error")) + if e and i not in errors: + errors[i] = e + + +def _status(resp) -> Optional[str]: + """A Responses object's end: its status, and why when it is incomplete.""" + if not isinstance(resp, dict) or not _name(resp.get("status")): + return None + why = resp.get("incomplete_details") + why = _name(why.get("reason")) if isinstance(why, dict) else None + return f"{resp['status']}: {why}" if why else resp["status"] + + +def _facts(path: str, body: bytes, streamed: bool) -> dict: + served, end, error, ends, errors = None, None, None, {}, {} + if not streamed: + got = json.loads(body) + if not isinstance(got, dict): + return {} + served, error = _provider(got), _an_error(got.get("error")) + _choices(got.get("choices"), ends, errors) + if got.get("type") == "message": + end = _name(got.get("stop_reason")) + elif got.get("object") == "response" or "output" in got: + end = _status(got) + else: + for line in body.decode("utf-8", "replace").splitlines(): + if not line.startswith("data:"): + continue + try: + ev = json.loads(line[5:].strip()) + except ValueError: + continue + if not isinstance(ev, dict): + continue + # the latest event that names one; routing facts that mark no single named endpoint leave none + served = _provider(ev) if _routing_facts(ev) is not None else (_provider(ev) or served) + kind = ev.get("type") + if kind == "error": # Messages' error event, or a Responses one + error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) + elif isinstance(ev.get("error"), dict): # an error chunk under a 200 + error = error or _an_error(ev["error"]) + if kind == "message_delta" and isinstance(ev.get("delta"), dict): + end = _name(ev["delta"].get("stop_reason")) or end + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): + resp = ev.get("response") + end = _status(resp) or end + error = error or _an_error(resp.get("error") if isinstance(resp, dict) else None) + _choices(ev.get("choices"), ends, errors) + if ends: + end = ends[min(ends)] if len(ends) == 1 else {str(i): ends[i] for i in sorted(ends)} + if error is None and errors: + i = min(errors) + error = dict(errors[i], choice=i) + return {k: v for k, v in (("served_by", served), ("end", end), ("error", error)) if v is not None} + + +def reply_facts(path: str, body: bytes, streamed: bool) -> dict: + """What a reply says besides what :func:`read_reply` reads, read apart from + it: the serving provider that answered (``served_by``), how it ended + (``end``: ``finish_reason``, ``stop_reason`` or a Responses status; one per + choice, by index, when there are several) and an error it holds though its + status was a success (``error``: code, type and message; a top-level one + whatever else the reply holds, else a choice's). A fact of the wrong type + is left out, and a reply that cannot be read gives none: nothing here can + stop a reply reaching its caller.""" + try: + return _facts(path, body, streamed) + except Exception: + return {} + + +def _plain(name) -> str: + """A provider's name or slug, compared: lower case, letters and digits.""" + return re.sub(r"[^a-z0-9]", "", str(name).split("/")[0].lower()) + + +def held(provider: str, only: List[str]) -> Optional[bool]: + """True when the provider that answered is one of ``only``, False when it shares nothing with them, None when + it cannot be told: a name is not always its slug.""" + name = _plain(provider) + slugs = {_plain(p) for p in only} + if not name or name in slugs: + return True if name else None + if any(s.startswith(name) or name.startswith(s) for s in slugs if s): + return None + return False + + def _request_tail(req: dict) -> dict: """The part of a request a reader needs: the model, how many messages, the last two, abridged, and the reasoning effort it asked for (chat's @@ -236,6 +492,37 @@ def _request_tail(req: dict) -> dict: "reasoning_effort": effort if isinstance(effort, str) else None} +def _image(data: str) -> Optional[str]: + """An inline image's type when its bytes show a whole one: its head, and its end after it.""" + try: + raw = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + return None + if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP": + return "image/webp" if int.from_bytes(raw[4:8], "little") + 8 <= len(raw) else None + if raw[:2] == b"BM": + return "image/bmp" if int.from_bytes(raw[2:6], "little") <= len(raw) else None + return next((t for head, tail, t in _WHOLE if raw.startswith(head) and tail in raw[len(head):]), None) + + +def _images(x) -> List[dict]: + """Each inline image a request carries, as Messages or chat sends one: its declared type, the type its bytes + show whole (None when they show none) and its size, never a byte of it.""" + if isinstance(x, list): + return [i for v in x for i in _images(v)] + if not isinstance(x, dict): + return [] + src, url = x.get("source"), x.get("image_url") + url = url.get("url") if isinstance(url, dict) else url + if x.get("type") == "image" and isinstance(src, dict) and src.get("type") == "base64" and isinstance(src.get("data"), str): + said, data = src.get("media_type"), src["data"] + elif isinstance(url, str) and url.startswith("data:") and ";base64," in url: + said, data = url[5:].split(";", 1)[0], url.split(";base64,", 1)[1] + else: + return _images(list(x.values())) + return [{"type": str(said)[:40] if said else None, "is": _image(data), "bytes": len(data)}] + + def _digest(obj) -> str: text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True) return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16] @@ -292,8 +579,9 @@ class _Pool: KEEP = 16 # idle connections kept per provider; more are closed - def __init__(self, base: str): + def __init__(self, base: str, timeout: float = TIMEOUT): u = urllib.parse.urlsplit(base) + self.timeout = timeout self.https = u.scheme == "https" self.host, self.port = u.hostname, u.port or (443 if self.https else 80) self.prefix = u.path.rstrip("/") @@ -309,7 +597,7 @@ def take(self, fresh: bool = False): kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection with self.lock: self.opened += 1 - return kind(self.host, self.port, timeout=TIMEOUT), False + return kind(self.host, self.port, timeout=self.timeout), False def give(self, conn, r): """Back to the pool when the reply was read whole and the provider @@ -355,7 +643,8 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, - ledger: Optional[str] = None, unit_caps: bool = True): + ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, + facts: Optional[bool] = None, timeout: float = TIMEOUT): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -373,8 +662,16 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s (:func:`request_fields`) - ``fidelity``'s, never a normal run's. ``ledger``: the file of the run's spending cap (``ledger``), shared by every relay of the job: a call is booked there before it is sent, - and not sent when it does not fit.""" + and not sent when it does not fit. ``serving``: by model, the serving + providers that may answer a request sent as that model (:func:`routing`), + put on every such request in place of the caller's own. ``facts``: a + pinned Chat Completions or Responses request asks OpenRouter for its + routing facts, where its reply names who served it; by default when + ``base`` is OpenRouter's. ``timeout``: the seconds one call may wait.""" self.fields = fields + self.serving = {m: list(s) for m, s in (serving or {}).items()} + host = (urllib.parse.urlsplit(base).hostname or "").lower() + self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) self.ledger = ledgers.Ledger(ledger) if ledger else None # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them self.unit_caps = unit_caps @@ -386,7 +683,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s self.models: Dict[str, str] = {} self.embedding_models: Dict[str, str] = {} self.lock = threading.Lock() - self.pool = _Pool(self.base) + self.pool = _Pool(self.base, timeout) self.server: Optional[ThreadingHTTPServer] = None os.makedirs(record_dir, exist_ok=True) @@ -437,8 +734,14 @@ def _said(self, e: Exception) -> str: text = text.replace(k, "[REDACTED]") return redact(text)[:200] - def _headers(self, caller) -> Dict[str, str]: + def _headers(self, caller, facts: bool = False) -> Dict[str, str]: + """``facts``: a pinned Chat Completions or Responses request to + OpenRouter, sent asking for its routing facts whatever the caller + said of them.""" out = forwarded(caller) + if facts: + out = {k: v for k, v in out.items() if k.lower() != "x-openrouter-metadata"} + out["X-OpenRouter-Metadata"] = "enabled" has = {k.lower() for k in out} # a name in any case is the same header: never send it twice for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ (("anthropic-version", ANTHROPIC_VERSION),) * (self.key_header == "x-api-key"): @@ -450,8 +753,13 @@ def _headers(self, caller) -> Dict[str, str]: out["Authorization"] = "Bearer " + self.key return out + def _asked(self) -> bool: + return asked(self.dir) is not None + def _forward(self, h: BaseHTTPRequestHandler): - """One call of a caller, ``h``: sent on, passed back, recorded.""" + """One call of a caller, ``h``: sent on, passed back, recorded. A caller's own call + refused for its context window also leaves STOP beside the records; one made once + another asked by STOP is refused unsent and unrecorded.""" bits = h.path.split("/", 2) if len(bits) < 3 or not _TAG.match(bits[1]): h.send_error(404, "no route: //v1/...") @@ -465,6 +773,7 @@ def _forward(self, h: BaseHTTPRequestHandler): req = {} asked = req.get("model") if isinstance(req, dict) else None side = False + routing_asked, pinned = None, None if isinstance(req, dict) and req: if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag): req["model"] = self.embedding_models[tag] @@ -477,7 +786,14 @@ def _forward(self, h: BaseHTTPRequestHandler): req["model"] = self.model if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"): req.setdefault("stream_options", {})["include_usage"] = True + pinned = self.serving.get(req["model"]) if isinstance(req.get("model"), str) else None + if pinned: + routing_asked = req.get("provider") + req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if not side and not self.models.get(tag) and self._asked(): + h.send_error(400, "the unit was asked to stop") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -498,8 +814,11 @@ def _forward(self, h: BaseHTTPRequestHandler): t0 = time.time() conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed + cut_off = None try: - conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers)) + # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself + facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) + conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: status, kind = 502, "application/json" @@ -516,28 +835,55 @@ def _forward(self, h: BaseHTTPRequestHandler): except Exception as e: # cut off mid-reply status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() + empty = _hollow(rest, body) if status == 200 and not streamed and h.command == "POST" \ + and not (isinstance(req, dict) and req.get("stream")) else None + if empty: + (status, inside), kind = empty, "application/json" + back = [(n, v) for n, v in back if n.lower() in ("retry-after", "retry-after-ms")] + body = json.dumps({"error": {"message": f"the endpoint answered 200 with {inside}"}}).encode() def record(body): rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None, - "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {}, + "model_asked": asked, "status": status, + "request": dict(_request_tail(req), sha256=_digest(req)) if isinstance(req, dict) else {}, "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True + if isinstance(req, dict) and req.get("provider") is not None: + # the routing as sent, and the caller's own the pin replaced: recorded copies, cleaned + rec["request"]["routing"] = _clean(req["provider"], (self.key, self.key.strip())) + if pinned and routing_asked is not None: + rec["request"]["routing_asked"] = _clean(routing_asked, (self.key, self.key.strip())) if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None - if status == 200: + if status == 200 or cut_off: rec["reply"] = read_reply(rest, body, streamed) - cost, how = self._cost(rec["model"], rec["reply"].get("usage")) + # who served it, how it ended, an error under the 200: beside what was read, never in its place + try: + rec["reply"].update({k: v for k, v in _clean(reply_facts(rest, body, streamed), (self.key, self.key.strip())).items() + if k not in rec["reply"]}) + except Exception: + pass + cost, how = (0.0, "failed") if rec["reply"].get("end") == "failed" and not rec["reply"].get("usage") \ + else self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - else: - rec["error"] = _said(status, body) + if status != 200: + rec["error"] = cut_off or _said(status, body) + sent = _images(req) if status in (400, 413, 422) else [] + if sent: + rec["images"] = sent + if empty: + rec["answered"] = 200 if booking is not None: rec["booked_usd"] = booked try: - if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) + if status == 200 or cut or empty: # what it cost; unknown (no token count, cut off), the booking is kept whole + whole = not (cut_off or rec.get("cost_from") == "failed") + known = cost if whole or (cost is not None and cost > booked) else None + self.ledger.settle(booking, known, rec["model"], + ((rec.get("reply") or {}).get("usage") or {}).get("output") if whole else None) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -545,6 +891,9 @@ def record(body): with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") + if not side and not self.models.get(tag) and overflowed(rec): + with open(os.path.join(self.dir, STOP), "a") as f: + f.write(OVERFLOW + "\n") h.send_response(status) if not any(n.lower() == "content-type" for n, _ in back): @@ -560,6 +909,8 @@ def record(body): while True: chunk = r.read1(65536) if not chunk: + if r.length: + raise http.client.IncompleteRead(b"", r.length) whole = True break parts.append(chunk) @@ -568,8 +919,8 @@ def record(body): h.wfile.flush() except OSError: pass # the caller went away; the record still completes - except Exception: - pass + except Exception as e: + status, cut, cut_off = 502, True, "the endpoint's reply was cut off: " + self._said(e) record(b"".join(parts)) # before the connection closes: the caller's end of stream else: record(body) # before the caller has the reply, so a reader after it finds the call @@ -693,11 +1044,26 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True -def failure(calls: List[dict]) -> Optional[tuple]: +def asked(record_dir: str) -> Optional[str]: + """What a guard's STOP in ``record_dir`` says, or None: none there, or only the context window's.""" + try: + with open(os.path.join(record_dir, STOP)) as f: + said = next((ln.strip() for ln in f.read().splitlines() or [""] if ln.strip() != OVERFLOW), None) + except OSError: + return None + return None if said is None else said or STOP + + +def failure(calls: List[dict], words: bool = True, agent: bool = False, serving: Optional[Dict[str, List[str]]] = None) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's). A call + succeeded (a failure the agent recovered from is not the unit's). + ``words`` False: the status alone, never the provider's text, which can quote + a gated task. ``agent``: the agent's last call over the model's context + window is ``ContextWindowExceededError``, its own, whatever side call followed, and a side call's 200 does not + recover the agent's own. A pinned call (``serving``) no endpoint took is + ``ApiProviderRouteError``. Calls are taken in the order they began. A call the run's spending cap refused is the unit's wherever it falls: once one is refused, every later one is.""" cut = next((c for c in calls if c.get("refused") == "cap"), None) @@ -706,26 +1072,81 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or calls[-1].get("status") == 200: + calls = _begun(calls) + ok = [i for i, c in enumerate(calls) if c.get("status") == 200] + after, side = max((i for i in ok if not (agent and calls[i].get("side"))), default=-1), max(ok, default=-1) + calls = [c for i, c in enumerate(calls) if i > after and c.get("status") != 200 and not (c.get("side") and i < side)] + if not calls: return None + own = next((c for c in reversed(calls) if not c.get("side")), None) if agent else None + if own is not None and overflowed(own): + said = (own.get("error") or "") if words else ("the endpoint answered 400 to the agent's call, longer than the model's " + "context window; its words are not kept, the benchmark's tasks being gated") + return CONTEXT_ERROR, said last = calls[-1] - s, said = last.get("status"), last.get("error") or "" + s = last.get("status") + said = (last.get("error") or "") if words else f"the endpoint answered {s}; its words are not kept, the benchmark's tasks being gated" + if (serving or {}).get(last.get("model")) and s in ROUTE_REFUSED: + return ROUTE_ERROR, _refusal(last, said, serving) if s == 429: return "RateLimitError", said if s in (401, 403): return "AgentAuthenticationError", said - if s == 404 and "model" in said.lower(): + if s == 404 and "model" in (last.get("error") or "").lower(): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said return "ApiError", said +def _begun(calls: List[dict]) -> List[dict]: + timed = all(isinstance(c.get("t0"), (int, float)) for c in calls) + return sorted(calls, key=lambda c: c["t0"]) if timed else list(calls) + + +def overflowed(call: dict) -> bool: + """A 400 for a request longer than the model's context window, by the provider's words.""" + return call.get("status") == 400 and bool(_CONTEXT.search(call.get("error") or "")) + + +def _refusal(call: dict, said: str, serving: Dict[str, List[str]]) -> str: + """A pinned route's refusal: its status first (route_dead reads it), the model, its pin, then ``said``.""" + text = f"{call.get('status')}: no endpoint of {call.get('model')} pinned to {', '.join(serving[call.get('model')])} took the call" + return f"{text}: {said}" if said else text + + +def routes_failure(routes: List[List[dict]], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """:func:`route_failure` of each route's calls on its own, the first found.""" + return next(filter(None, (route_failure(calls, serving, words) for calls in routes)), None) + + +def route_dead(exc: Optional[tuple]) -> bool: + """A pinned route with no endpoint at all (404): every later unit would fail the same way.""" + return bool(exc) and exc[0] == ROUTE_ERROR and str(exc[1]).startswith("404: ") + + +def route_failure(calls: List[dict], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """A pinned route (``serving``) that failed a unit, among one route's ``calls``: a model whose last call was + refused, or a call its reply says another provider served, by a name that reads as one.""" + last = {} + for c in _begun(calls): + last[c.get("model")] = c + by = (c.get("reply") or {}).get("served_by") + if serving.get(c.get("model")) and isinstance(by, str) and _PROVIDER.fullmatch(by) and held(by, serving[c.get("model")]) is False: + return ROUTE_ERROR, f"{c.get('status')}: {c.get('model')} pinned to {', '.join(serving[c.get('model')])} was served by {by}" + for c in last.values(): + if serving.get(c.get("model")) and c.get("status") in ROUTE_REFUSED: + said = (c.get("error") or "") if words else \ + f"the endpoint answered {c.get('status')}; its words are not kept, the benchmark's tasks being gated" + return ROUTE_ERROR, _refusal(c, said, serving) + return None + + # -- the relay as a process of its own: a container backend's sidecar ---------- def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME] - [--public URL] [--prices JSON] [--ledger FILE]``. + [--public URL] [--prices JSON] [--ledger FILE] [--served-by MODEL=SLUG[,SLUG...] ...]``. ``--base`` is the provider's endpoint, ``--route`` a route sent as another model (a benchmark's judge), ``--public`` the address ``url`` hands out. @@ -747,7 +1168,26 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", + help="the serving providers that may answer a request sent as MODEL, in order, and no other; " + "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " + "or --routing given") + ap.add_argument("--routing", action="store_true", + help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") + ap.add_argument("--timeout", type=float, default=TIMEOUT, help="seconds one call may wait for the provider's reply") a = ap.parse_args(argv) + serving, bad = parse_served_by(a.served_by, "") + if serving and not bad: + host = (urllib.parse.urlsplit(a.base).hostname or "").lower() + if not (a.routing or host == "openrouter.ai" or host.endswith(".openrouter.ai")): + bad.append(f"{a.base} routes no model to several serving providers (OpenRouter's does; --routing says another does)") + sent = {a.model} | {r.partition("=")[2] for r in a.route} | {(s.partition("=")[2] or s) for s in a.side_model} + stray = [m for m in serving if a.model and m not in sent] + if stray: + bad.append(f"{', '.join(stray)}: a model this relay does not send (it sends {', '.join(sorted(x for x in sent if x))})") + if bad: + print(json.dumps({"error": "--served-by: " + "; ".join(bad)}), flush=True) + return 2 try: with open(a.key_file) as f: key = f.read().strip() @@ -759,7 +1199,8 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, + timeout=a.timeout) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 236c8b6..ce5854a 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -51,6 +51,7 @@ """ from __future__ import annotations +import fcntl import hashlib import json import os @@ -66,7 +67,7 @@ from collections import Counter from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from contextlib import contextmanager -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact @@ -100,7 +101,8 @@ def driver_path() -> str: PROVIDERS = { "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"}, - "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"}, + # routing: one model is served by several providers, and a request may say which may answer (--served-by) + "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY", "routing": True}, "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"}, "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } @@ -113,6 +115,10 @@ def driver_path() -> str: _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) +HELD: dict = {} +LIVE: dict = {} + + class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" @@ -161,6 +167,19 @@ def units_file(path: str) -> dict: return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} +def served_by(given, model: str) -> Dict[str, List[str]]: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable: by model, the + serving providers that may answer it, in the order they are tried, and no + other (OpenRouter's slugs, as a model's page gives them: ``deepinfra``, + ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused as + ``relay.parse_served_by`` finds it wrong: the check the run and the + relay's own command make too.""" + out, bad = relay.parse_served_by(given, model) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + return out + + def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit" @@ -346,7 +365,37 @@ def get(self, name: str) -> Optional[dict]: return {"kind": kind, "value": text} if text else None +def resolve_bench_options(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + which=shutil.which) -> Tuple[dict, dict]: + """The benchmark's options for a run on ``dataset`` and what their values need; refused before anything starts.""" + vals, bad = definitions.bench_options_of(bench, dataset, given, setup) + if bad: + raise Refused("; ".join(bad)) + needs = definitions.option_needs(bench, vals) + lacking = hostreq.problems(bench, needs["host"], which) + if lacking: + raise Refused(f"{', '.join(f'{k}={v}' for k, v in vals.items())} needs on this machine " + + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) + else ". Run it on a machine that has it")) + return vals, needs + + +def pins_of(bench: dict, other: dict, provider: str, judge_model: Optional[str]) -> Dict[str, List[str]]: + """{model as sent: [provider slug]}: the original setup's pins and the official judge's, on a provider that pins.""" + if provider not in definitions.PIN_PROVIDERS: + return {} + pins = {m: list(o) for m, o in (other.get("pins") or {}).items()} if other.get("_kind") == "upstream" else {} + j = (bench.get("score") or {}).get("judge") or {} + only = (j.get("pin") or {}).get(provider) + if only and judge_model and judge_model == providers.served_as(j["model"], provider): + pins[judge_model] = list(only) + return pins + + class Run: + call_seconds: float = relay.TIMEOUT + def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -356,7 +405,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -402,7 +452,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas agents on the local backend). Before the first unit the container constraints are checked on the engine (where those containers run), and the scope's on systemd: on the local backend a cap no scope holds is refused, on the container - backend its drivers then run without one.""" + backend its drivers then run without one. + ``served_by``: by model, the serving providers that may answer it + (:func:`served_by`), put on every request sent as that model by the + run's relays (``relay.routing``) and recorded in the job's config. + Refused when the provider routes no model to several (only + ``openrouter`` does), or when it names a model the run does not call.""" backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") @@ -526,6 +581,24 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) + self.serving = dict(served_by or {}) + if self.serving: + bad = relay.serving_problems(self.serving) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + self.serving = {m: list(s) for m, s in self.serving.items()} + if not PROVIDERS.get(provider, {}).get("routing"): + raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " + "among; the option needs a provider that routes a model to several (openrouter)") + # a control calls no model of its own and plays no user: only its judge and an option's models + called = ([m for m in (self.judge_model, self.embed_model, self.rerank_model) if m] if other.get("_kind") == "control" + else self.models_called()) + stray = [m for m in self.serving if m not in called] + if stray: + raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + + (", ".join(called) or "no model")) + self.route_pins = pins_of(bench, other, provider, self.judge_model) + self.serving.update(self.route_pins) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -541,6 +614,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + self.score_seconds = (bench.get("limits") or {}).get("score_seconds") or DRIVER_SECONDS + self.call_seconds = (bench.get("limits") or {}).get("call_seconds") or relay.TIMEOUT self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -608,8 +683,25 @@ def driver(self) -> jsonl.Proc: return d def start(self) -> str: + lck, mine = os.path.join(self.job_dir, ".work", "job.lck"), object() + try: + return self._start(lck, mine) + finally: + if LIVE.get(lck) is mine: + del LIVE[lck] + + def _start(self, lck, mine) -> str: t0 = time.time() os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True) + held = HELD.get(lck) or open(lck, "a") + try: + if LIVE.setdefault(lck, mine) is not mine: + raise BlockingIOError + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused(f"another run under the job name {self.name} is running: " + "start this one once it ends, or under another name") from None + HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) @@ -682,7 +774,7 @@ def start(self) -> str: # a shard of a list splits the listed units: they are its whole shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, - repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, + repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -755,7 +847,8 @@ def work_unit(iur): self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) self.capped = bool(budget and budget["stopped"]) - job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget) + job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget, + served=job.served(self.benchmark_calls(), getattr(self, "serving", None))) errored = sum(1 for x in results if x.get("exception_info")) self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results)) # the units' stage says why it ended early: a line of its own would be a stage that never starts @@ -807,11 +900,14 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, job's ``retried/`` (kept whole, its calls and its cost with it, where no reader takes it for a trial of its own), its second written under the trial's own name and counted (into ``done``, as each ends). Nothing - is run again once the run is stopped or halted; the spending cap goes - on counting every call.""" + is run again once the run is stopped or halted, nor a unit a guard's + stop asked to stop; the spending cap goes on counting every call.""" + if (self.bench.get("limits") or {}).get("run_again") is False: + return by_name = {self.trial_name(*iur): iur for iur in work} todo = [by_name[r["trial_name"]] for r in results - if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name] + if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name + and relay.asked(os.path.join(self.job_dir, r["trial_name"], ".work", "relay")) is None] if not todo: return self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time") @@ -917,6 +1013,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False, + serving=self.serving, timeout=self.call_seconds, key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start() def runtimes_needed(self) -> List[str]: @@ -955,6 +1052,11 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home") + earlier = os.path.join(trial, ".work", f"earlier.{time.time_ns()}") + for part in (os.path.join(".work", "relay"), "result.json", "verifier", "agent"): + if os.path.lexists(os.path.join(trial, part)): + os.makedirs(earlier, exist_ok=True) + os.replace(os.path.join(trial, part), os.path.join(earlier, os.path.basename(part))) os.makedirs(ws, exist_ok=True) # The workspace is a repository of its own: agents look up to the git root # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept @@ -1027,17 +1129,17 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.score_seconds + (float(step.get("seconds") or self.turn_seconds) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.score_seconds, **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e)) + raise _Ended(_kind(e, "VerifierError"), str(e)) except _Ended as e: exc = (e.kind, e.said) traj = traj or e.trajectory @@ -1056,17 +1158,27 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: for ln in getattr(rl, "mcp_listeners", None) or []: ln.stop() calls = rl.calls(tag) if rl is not None else [] - failed = relay.failure(calls) + words = not self.bench.get("gated") + failed = relay.failure(calls, words, agent=True, serving=getattr(self, "route_pins", None)) if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is # the endpoint's, and a cap's refusal is the cap's failed = None - if not failed and rl is not None and exc is not None and exc[0] == "VerifierError": + if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") + or self.upstream and exc and ": VerifierError: " in exc[1]): + failed = None + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and (score or {}).get("stopped") not in (None, "", relay.OVERFLOW) \ + and rl is not None and relay.asked(rl.dir) is not None \ + and [c.get("status") for c in relay._begun(calls) if not c.get("side")][-1:] == [429]: + failed = None + if not failed and rl is not None and getattr(self, "route_pins", None): + failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) + if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): # the judge's or the simulated user's model failing (a 429, a refused request) is the # unit's infrastructure, as the agent's own would be - not a miss counted against it - failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) + failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}"), words) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score = None @@ -1093,7 +1205,10 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), - environment=environment or "", + environment=environment or "", serving=getattr(self, "serving", None), + # who served the unit: its agent's calls and its judge's and simulated user's routes + served_calls=calls + ([c for side in ("judge", "simulator") for c in rl.calls(f"{tag}-{side}")] + if rl is not None else []), image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" @@ -1164,7 +1279,7 @@ def note_refusal(self, exc: Optional[tuple]) -> None: if getattr(self, "_halt", None) is None: return with self._said: - if exc and exc[0] in REFUSALS: + if exc and (exc[0] in REFUSALS or relay.route_dead(exc)): self._refused.append(exc) if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None: self.halted = exc @@ -1278,6 +1393,16 @@ def ledger_file(self) -> Optional[str]: led = getattr(self, "ledger", None) return led.path if led is not None else None + def benchmark_calls(self) -> List[dict]: + """The calls of the benchmark's own relay (an option's embeddings or reranking), every route of it.""" + folder = os.path.join(self.job_dir, ".work", "relay-benchmark") + calls = [] + for name in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + if name.endswith(".jsonl"): + with open(os.path.join(folder, name)) as f: + calls += [json.loads(line) for line in f if line.strip()] + return calls + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the @@ -1302,7 +1427,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = PROVIDERS[self.provider].get("key_header", "authorization") if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file()).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1331,7 +1457,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds if self.call_seconds != relay.TIMEOUT else None, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1850,8 +1977,8 @@ def send(op, **kw): def drive_upstream(self, unit, ws, tag, transcript, rl): """The setup's unit: the driver runs the upstream's own answerer on it (``answer``), asked as ``asked`` through the unit's relay, which sends - the run's model in its place. A reply with no text - a reasoning model - that spent the upstream's token budget thinking - is said as such.""" + the run's model in its place. A last reply with no text or tool call - a reasoning model + that spent the upstream's token budget thinking - is said as such; a unit a guard stopped is cancelled.""" o = self.other try: # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes @@ -1860,7 +1987,18 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) except jsonl.Failed as e: calls = rl.calls(tag) - silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] + stop = relay.asked(rl.dir) + if stop and not _kind(e, None): + raise _Ended(CANCELLED, f"a guard asked the unit to stop ({stop[:200]}); {o['label']}: {e}", + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + own = [c for c in relay._begun(calls) if not c.get("side")][-1:] + failed = [((c["reply"].get("error") or {}).get("code") or "no code") for c in own if (c.get("reply") or {}).get("end") == "failed"] + kind = _kind(e, "ApiError" if failed else None) + if kind: + raise _Ended(kind, f"{o['label']}: {e}" + "".join(f"; {self.model}'s last reply failed ({c})" for c in failed), + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + silent = [c for c in own if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip() + and not (c.get("reply") or {}).get("tool_calls")] if silent: u = (silent[-1].get("reply") or {}).get("usage") or {} thought = u.get("reasoning") or 0 @@ -1875,6 +2013,12 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") +def _kind(e, otherwise): + """A driver's own EnvironmentSetupError, VerifierError, CancelledError or CauseUnknownError keeps its kind.""" + m = re.match(r"\w+: (EnvironmentSetupError|VerifierError|CancelledError|CauseUnknownError): ", str(e)) + return m.group(1) if m else otherwise + + class _Ended(Exception): def __init__(self, kind, said, trajectory=None): super().__init__(said) diff --git a/app/site/mkdocs.yml b/app/site/mkdocs.yml index 2af82be..94bf3ae 100644 --- a/app/site/mkdocs.yml +++ b/app/site/mkdocs.yml @@ -86,6 +86,7 @@ nav: - Add manually: benchmarks/add-manually.md - Add with an assistant: benchmarks/add-with-assistant.md - OSWorld1 machine setup: benchmarks/osworld1-machine.md + - OSWorld2 machine setup: benchmarks/osworld2-machine.md - Agents: - Supported agents: agents/supported/README.md - Add manually: agents/add-manually.md diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index d20fd5b..395952f 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -69,6 +69,7 @@ r.model_choice = choice; if (paperListed(r)) r.model = ''; if (choice !== 'any') { delete r.judge_model; delete r.simulator_model; } + if (choice !== was) delete r.served_by; // its models change with it: a pin belongs to the model it was set for if (lb && was === 'leaderboard' && choice !== 'leaderboard') { r.bench_options = {...(r.bench_options || {})}; for (const k of Object.keys(lb.bench_options || {})) delete r.bench_options[k]; @@ -79,6 +80,34 @@ } return r; } + // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are + // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. + const SERVED_ROLES = ['model', 'simulator', 'judge']; + function servedRoles(r) { + const b = standardBench(r); + if (!b || r.provider !== 'openrouter' || standardKeyless(r)) return []; + return SERVED_ROLES.filter(k => k === 'model' || (k === 'simulator' && b.simulator) || (k === 'judge' && b.judge)); + } + // A field's text, "deepinfra, together", as the pin's list, every part kept for the server to check ("a,,b" is + // refused in sight, as the option refuses it); a field with nothing in it pins nothing. + const servedList = text => { const parts = String(text || '').split(',').map(s => s.trim()); return parts.some(Boolean) ? parts : []; }; + // A model field's role, and its model changed: the pin set for the model before it goes with it. + const SERVED_FIELD = {model: 'model', simulator_model: 'simulator', judge_model: 'judge'}; + function setModelOf(r, field, value) { + if (SERVED_FIELD[field] && r[field] !== value && r.served_by) setServed(r, SERVED_FIELD[field], []); + r[field] = value; + return r; + } + // The model a field's Served by lists the providers of: the one the run sends, as the server compiled it + // (recipe.py, served_models: a paper model's served name, an official one's with its maker), else the one shown. + const servedModelOf = (role, shown, compiled) => ((compiled || {}).served_models || {})[role] || shown || ''; + function setServed(r, role, slugs) { + const list = Array.isArray(slugs) ? slugs : servedList(slugs); + r.served_by = {...(r.served_by || {})}; + if (list.length) r.served_by[role] = list; else delete r.served_by[role]; + if (!Object.keys(r.served_by).length) delete r.served_by; + return r; + } function createStandard(benchmark = '', other = '') { const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0]; const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [], @@ -94,7 +123,7 @@ const b = standardBench({benchmark: key}); if (!b) return r; const was = standardBench(r); - if (was !== b) r.model_choice = 'any'; + if (was !== b) { r.model_choice = 'any'; delete r.served_by; } r.benchmark = key; const kept = was === b ? datasetsOf(r).filter(id => b.datasets.some(d => d.id === id)) : []; r.datasets = kept.length ? kept : [b.datasets[0].id]; @@ -141,6 +170,11 @@ // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); + // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) + if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) + for (const role of SERVED_ROLES) { const v = raw.served_by[role]; + // kept as saved, a slug the server refuses included: a pin is refused in sight, never dropped in silence + if (Array.isArray(v) && v.length) setServed(r, role, v.map(s => typeof s === 'string' ? s.trim() : String(s))); } r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; else r.model = bareModel(r.model); @@ -217,7 +251,8 @@ const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel); const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r); // The Model step's choice into the recipe: its provider, and for a named endpoint its id. - function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); } + // a provider that routes no model to several takes no Served by: a pin made for OpenRouter is dropped with it + function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); if (r.provider !== 'openrouter') delete r.served_by; } // Why Run cannot start this recipe here, or '': its provider is not one of those. A recipe naming one keeps it, // said, until another is chosen - never switched behind the person's back. The server refuses the same // (web/execute.py, provider_problem). @@ -759,5 +794,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,setModelOf,servedModelOf,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index e37e016..af45dec 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -246,3 +246,8 @@ #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +/* Served by: under its model field, set in from it: the typed list, then the providers to add from */ +#nullius-builder .std-served { margin: 10px 0 22px 18px; padding: 2px 0 2px 14px; border-left: 2px solid var(--nb-line); font-size: 14px; gap: 8px; } +#nullius-builder .std-served label { display: grid; gap: 6px; } +#nullius-builder .std-served select { max-width: 420px; } +@media(max-width:520px) { #nullius-builder .std-served { margin-left: 6px; padding-left: 10px; } } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index f4eb338..ff3793e 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -114,7 +114,9 @@ }; // The standard's recipe as the server compiles it from the definitions (POST // /new-run/runner). Nothing typed here is a command; Run sends the choices. - let runnerOut = null, runnerKey = '', runnerTimer; + let runnerOut = null, runnerKey = '', runnerTimer, runnerOutKey = ''; + // the server's compile of the recipe as it is now, or null while a newer one is asked + const compiledNow = () => runnerOutKey === JSON.stringify(r()) ? runnerOut : null; // Switching runner keeps the recipe being edited, one per runner, for as long as the page is open: switching back finds it as it was. const keptByRunner = {}; // Nullius's own mark for the standard; Harbor's for Harbor @@ -602,12 +604,13 @@ return `

Choose the model.

` + modelChoiceCards(rec) + providerField(rec, C.PROVIDERS) + field('Model', 'model', rec.model, {list:'nb-models', placeholder:paper && C.paperListed(rec) ? 'Choose a paper model' : 'Search or enter a model ID'}) + + servedField(rec, 'model', rec.model) // the benchmark's other models: the user simulator its conversations need, the judge that grades - each its official one // unless named; like the paper, the official one; like the leaderboard, its own and the options it fixes + (b.simulator ? (paper ? fixed('Model for the user simulator', lb && lb.simulator || b.simulator, lb && lb.simulator ? 'leaderboard' : 'official') : field('Model for the user simulator', 'simulator_model', rec.simulator_model || '', {list: 'nb-models', placeholder: b.simulator + ' (official)', - hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) : '') + hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) + servedField(rec, 'simulator', (!paper && rec.simulator_model) || lb && lb.simulator || b.simulator) : '') + (b.judge ? (paper ? fixed('Model for the judge', lb && lb.judge || b.judge, lb && lb.judge ? 'leaderboard' : 'official') : field('Model for the judge', 'judge_model', rec.judge_model || '', {list: 'nb-models', placeholder: b.judge + ' (official)', - hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) : '') + hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) + servedField(rec, 'judge', (!paper && rec.judge_model) || lb && lb.judge || b.judge) : '') + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, v, 'leaderboard')).join('') : '') + keyLine(rec); } @@ -637,7 +640,8 @@ out = await res.json(); } catch (e) { out = {errors: [{step: REVIEW, message: 'The recipe could not be compiled here: ' + e}], notes: [], review: []}; } if (key !== runnerKey) return; - runnerOut = out; + runnerOut = out; runnerOutKey = key; + if (step === S.model) fillServed(); if (step === REVIEW) renderForm(); refreshPlan(); }, 120); @@ -646,6 +650,41 @@ const select = (label,key,value,options) => ``; const fixed = (label,value,whose='official') => ``; const note = (title,text) => `
${title}${text}
`; + // Served by: under a model field, on OpenRouter alone, the serving providers that may answer it, in order (recipe.py, + // served_by); typed as slugs, or added from the model's providers (POST /new-run/endpoints) + const SERVED_NAME = {model: 'model', simulator: 'user simulator', judge: 'judge'}; + // its list: the providers of the model the run sends (the server's served_models), not of the name shown + const servedField = (rec, role, model) => !C.servedRoles(rec).includes(role) ? '' : + `
` + + `
`; + const servedCache = new Map(); + function servedList(model) { + let entry = servedCache.get(model); + if (!entry || Date.now() - entry.at > 300000) { + entry = {at: Date.now(), promise: nulliusFetch('/new-run/endpoints', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': schema.csrf}, body: JSON.stringify({provider: 'openrouter', model})}) + .then(x => x.json()).then(x => ({...x, endpoints: Array.isArray(x.endpoints) ? x.endpoints : []})) + .catch(() => ({ok: false, endpoints: [], message: "Provider list unavailable. Enter a provider's slug by hand."}))}; + servedCache.set(model, entry); + } + return entry.promise.then(x => { if (!x.ok) servedCache.delete(model); return x; }); + } + function fillServed() { + root.querySelectorAll('select[data-served-add]').forEach(async select => { + // the model the run sends, once the server has compiled it: a list asked under another name is asked again + const sent = C.servedModelOf(select.dataset.servedAdd, select.dataset.servedShown, compiledNow()); + if (sent !== select.dataset.servedModel) { select.dataset.servedModel = sent; select.length = 1; delete select.dataset.servedFilled; } + if (select.dataset.servedFilled) return; + select.dataset.servedFilled = 'true'; + const model = select.dataset.servedModel; + if (!model) return; + const got = await servedList(model); + if (!select.isConnected || select.dataset.servedModel !== model) return; + const first = select.options[0]; + if (!got.ok || !got.endpoints.length) { first.textContent = got.ok ? 'No provider listed: type a slug' : got.message; return; } + first.textContent = 'Add a provider…'; + for (const e of got.endpoints) { const o = document.createElement('option'); o.value = e.slug; o.textContent = e.name + ' · ' + e.slug + (e.quantization ? ' · ' + e.quantization : ''); select.append(o); } + }); + } function customRows(agent = '') { // '' lists the shared entries (every agent, and the other owners); an id lists that agent's own --ak. @@ -731,7 +770,7 @@ const rows=own?[...((runnerOut&&runnerOut.review)||[]),...(C.isStandard(rec)?[]:[[S.where,'Where',C.whereLabel(rec)]])]:C.review(rec); html='

Review the experiment.

'+rows.map(([i,t,v])=>`
${t}${esc(v)}
${button('Edit',`data-step="${i}"`)}
`).join('')+'
'+(schema.local?'
':'')+runPanel(); } - $('#nb-form').innerHTML=html+'

';applyPick(); + $('#nb-form').innerHTML=html+'

';applyPick();fillServed(); if(step===REVIEW&&schema.local){drawPrereq();checkPrereq();} $('#nb-back').hidden=step===S.benchmark; $('#nb-next').textContent=step===REVIEW?'Export recipe JSON':stepName(steps[nextShown(step,1)],nextShown(step,1))+' →'; @@ -949,6 +988,7 @@ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} + if(d.servedBy!==undefined){C.setServed(rec,d.servedBy,el.value);refreshPlan();return;} if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; const value=el.type==='number'?(el.value===''?'':Number(el.value)):el.value; @@ -958,6 +998,7 @@ else if(!d.providerModel&&['model','userModel','evalModel'].includes(d.field)&&/^(openrouter|openai)\//.test(String(value))){rec.provider=C.providerOf(rec,value);rec[d.field]=C.bareModel(value); renderForm();refreshPlan();const f=root.querySelector(`[data-field="${d.field}"]`);if(f){f.focus();f.setSelectionRange(f.value.length,f.value.length);}return;} // the provider the prefix named, and whether it is set else if(d.field==='provider'){C.setChoice(rec,value);renderForm();refreshPlan();return;} // whether it is set in Settings, and the names a runner's models are sent as, follow the provider + else if(['model','simulator_model','judge_model'].includes(d.field))C.setModelOf(rec,d.field,value); // the field's pin goes with its model else rec[d.field]=value; if(el.list?.id==='nb-paths')completeDirectory(el.value); }else if(d.minutes){C.setPhaseMinutes(rec,d.minutes,el.value===''?'':Number(el.value));const row=el.closest('tr');if(row){const m=row.querySelector('.nb-mono');const t=C.effectiveTimeouts(rec);const p=t&&t.phases.find(x=>x.key===d.minutes);if(m&&p)m.textContent='×'+p.multiplier;}refreshPlan();return;} @@ -979,6 +1020,11 @@ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} else if(d.stdUnsandboxed!==undefined){r().unsandboxed=el.checked;refreshPlan();} + // a provider added to a Served by, after the ones already there; the field shows the list, in its order + else if(d.servedAdd!==undefined){const rec=r(),role=d.servedAdd,now=(rec.served_by||{})[role]||[];if(el.value&&!now.includes(el.value))C.setServed(rec,role,[...now,el.value]);el.value='';const f=root.querySelector(`[data-served-by="${role}"]`);if(f)f.value=((r().served_by||{})[role]||[]).join(', ');refreshPlan();} + // a model chosen: its Served by lists that model's providers; the keyboard stays on the field the new form holds + else if(['model','simulator_model','judge_model'].includes(d.field)&&C.servedRoles(r()).length){renderForm();refreshPlan(); + requestAnimationFrame(()=>{const f=root.querySelector(`[data-field="${d.field}"]`);if(f&&document.activeElement!==f)f.focus();});} else if(d.stdVersion!==undefined){C.setStandardBenchmark(r(),el.value);render();} // an option at its default is left out: the recipe names only what was changed. A select's value can show or hide // the options that apply under it (`when`): the step is drawn again and the focus goes back to the select, or to the diff --git a/app/static/catalogue/marks.svg b/app/static/catalogue/marks.svg index cc000c8..1576916 100644 --- a/app/static/catalogue/marks.svg +++ b/app/static/catalogue/marks.svg @@ -30,4 +30,7 @@ + + + diff --git a/app/tests/test_catalog.py b/app/tests/test_catalog.py index 24345d3..6b67099 100644 --- a/app/tests/test_catalog.py +++ b/app/tests/test_catalog.py @@ -44,7 +44,7 @@ def test_tabs_are_alphabetical_and_lanes_are_not_benchmarks(self): self.assertEqual(titles, sorted(titles, key=catalog.sort_key)) # what has a usable route is listed, whoever made it and whether or not it has run listed = {x.key for x in cat.listed_benchmarks} - self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "osworld1", "swe-bench-verified", "tau3-bench", "terminal-bench"]) + self.assertEqual(sorted(listed), ["gaia", "gsm8k", "humaneval", "locomo", "longmemeval", "osworld1", "osworld2", "swe-bench-verified", "tau3-bench", "terminal-bench"]) self.assertEqual((cat.benchmark("longmemeval").origin, cat.benchmark("longmemeval").label), ("official", "Official")) self.assertEqual(cat.benchmark("tau3-bench").label, "Official") # τ³ files as "t3…", before "te…": a reader looking under T finds it diff --git a/app/tests/test_dependency_surface.py b/app/tests/test_dependency_surface.py index 12b6b2e..d6cbd84 100644 --- a/app/tests/test_dependency_surface.py +++ b/app/tests/test_dependency_surface.py @@ -9,6 +9,7 @@ viewer independent of the runtime. """ +import fnmatch import importlib.util import json import os @@ -80,6 +81,26 @@ def test_every_backend_package_is_shipped_once(self): self.assertEqual({p for p in packages if p == "nullius" or p.startswith("nullius.")}, source) self.assertEqual({p.split(".")[0] for p in packages}, {"nullius", "_nullius_app"}) + def test_every_file_a_bundled_definition_runs_is_shipped(self): + """Every file a host's snapshot takes of a bundled entry is package data too.""" + from nullius.standard import payload + shipped = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["setuptools"]["package-data"] + + def covered(package, relative): + parts = relative.split("/") + return any(len(g.split("/")) == len(parts) and all(map(fnmatch.fnmatchcase, parts, g.split("/"))) + for g in shipped[package]) + + base = APP / "nullius" + entries = [("nullius.catalogue", f"{kind}/{e.name}", payload._entry(e)) + for kind in ("agents", "benchmarks", "capabilities") + for e in sorted((base / "catalogue" / kind).iterdir()) if e.is_dir() and e.name != "__pycache__"] + entries += [("nullius.standard", f"kits/{e.name}", payload._entry(e, kit=True)) + for e in sorted((base / "standard" / "kits").iterdir()) if (e / "standard.json").is_file()] + missing = [f"{where}/{name}" for package, where, files in entries for name in files + if not covered(package, f"{where}/{name}")] + self.assertEqual(missing, []) + def test_every_requirement_is_on_the_allowlist(self): for req in requirements(): with self.subTest(req): diff --git a/app/tests/test_osworld2_api_logs.py b/app/tests/test_osworld2_api_logs.py new file mode 100644 index 0000000..1c1815b --- /dev/null +++ b/app/tests/test_osworld2_api_logs.py @@ -0,0 +1,384 @@ +"""OSWorld2's api_logs: each image kept once, every file given back byte for byte, a failure leaving each whole.""" +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +LIMIT = 20 + + +def module(name, path, **env): + spec = importlib.util.spec_from_file_location(name, path) + loaded = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, **env): + spec.loader.exec_module(loaded) + return loaded + + +api_logs = module("osworld2_api_logs", os.path.join(HERE, "api_logs.py")) + + +def jpeg(n, size=3000): + return b"\xff\xd8\xff\xe0" + bytes([n % 256]) * size + + +def png(n): + return b"\x89PNG\r\n\x1a\n" + bytes([n % 256]) * 2000 + + +def b64(data): + return base64.b64encode(data).decode() + + +def call(i, window=3): + """One model call as the agent logs it: its window's screenshots, newest last, in Anthropic blocks.""" + blocks = [{"type": "text", "text": f"step {i} é中 \"quoted\" \\ /"}] + blocks += [{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64(jpeg(j))}} + for j in range(max(0, i - window + 1), i + 1)] + return {"model": "m", "request_body": {"system": "s", "messages": [{"role": "user", "content": blocks}], + "temperature": 0.6, "max_tokens": 1e-05}, + "response_body": {"content": [{"type": "text", "text": "pyautogui.click(1, 2)"}], "usage": None}, + "request_time": 1700000000000 + i, "response_time": 1700000000500 + i} + + +def odd(): + """Strings pack must leave inline, beside a PNG it packs.""" + text = b64(b"plain text, no image " * 40) + loose = b64(jpeg(7, 600)) + letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + loose = loose[:-3] + letters[letters.index(loose[-3]) | 1] + "==" + return {"a": b64(png(1)), "b": text, "c": loose, "d": b64(jpeg(8, 30)), "e": "x" + b64(jpeg(9)), + "f": [b64(jpeg(1)), b64(png(1))]} + + +def read(path): + with open(path, "rb") as f: + return f.read() + + +def digest(folder): + return {name: hashlib.sha256(read(os.path.join(folder, name))).hexdigest() + for name in sorted(os.listdir(folder)) if os.path.isfile(os.path.join(folder, name))} + + +class Logs(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.dir = os.path.join(self.tmp, "api_logs") + os.makedirs(self.dir) + for i, entry in enumerate([call(i) for i in range(6)] + [odd()]): + with open(os.path.join(self.dir, f"{i + 1:03d}_{1700000000000 + i}.json"), "w") as f: + json.dump(entry, f, ensure_ascii=False) + os.chmod(os.path.join(self.dir, "001_1700000000000.json"), 0o640) + os.utime(os.path.join(self.dir, "001_1700000000000.json"), ns=(1, 2_000_000_000)) + self.before = digest(self.dir) + + def images(self): + return sorted(os.listdir(os.path.join(self.dir, api_logs.IMAGES))) + + def test_each_image_is_written_once_and_every_file_restores_byte_for_byte(self): + done = api_logs.pack(self.dir) + self.assertEqual((done["files"], done["packed"], done["images"], done["written"]), (7, 7, 7, 7)) + self.assertLess(done["after"] * 5, done["before"]) + names = self.images() + self.assertEqual(len(names), 7) + self.assertEqual(sum(n.endswith(".png") for n in names), 1) + for name in names: + with open(os.path.join(self.dir, api_logs.IMAGES, name), "rb") as f: + self.assertEqual(hashlib.sha256(f.read()).hexdigest(), name.split(".")[0]) + with open(os.path.join(self.dir, "006_1700000000005.json"), encoding="utf-8") as f: + entry = json.load(f) + self.assertEqual(entry["_packed_images"], "images") + links = [b["source"]["data"] for b in entry["request_body"]["messages"][0]["content"] if b["type"] == "image"] + self.assertEqual(links, [f"images/{hashlib.sha256(jpeg(j)).hexdigest()}.jpg" for j in (3, 4, 5)]) + with open(os.path.join(self.dir, "007_1700000000006.json"), encoding="utf-8") as f: + kept = json.load(f) + self.assertEqual({k: v == odd()[k] for k, v in kept.items() if k != "_packed_images"}, + {"a": False, "b": True, "c": True, "d": True, "e": True, "f": False}) + self.assertEqual(api_logs.restore(self.dir), {"restored": 7, "failed": 0}) + self.assertEqual({k: v for k, v in digest(self.dir).items()}, self.before) + st = os.stat(os.path.join(self.dir, "001_1700000000000.json")) + self.assertEqual((st.st_mode & 0o777, st.st_mtime_ns), (0o640, 2_000_000_000)) + + def test_running_either_again_changes_nothing(self): + api_logs.pack(self.dir) + packed, images = digest(self.dir), digest(os.path.join(self.dir, api_logs.IMAGES)) + again = api_logs.pack(self.dir) + self.assertEqual((again["packed"], again["written"]), (0, 0)) + self.assertEqual((digest(self.dir), digest(os.path.join(self.dir, api_logs.IMAGES))), (packed, images)) + api_logs.restore(self.dir) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 0}) + self.assertEqual(digest(self.dir), self.before) + + def test_a_file_pack_did_not_mark_is_left_alone_by_both(self): + name = hashlib.sha256(jpeg(0)).hexdigest() + ".jpg" + shapes = {"x_link.json": json.dumps({"a": "images/" + name, "b": b64(jpeg(0))}), + "x_list.json": json.dumps([b64(jpeg(0))]), + "x_empty.json": "{}", + "x_broken.json": '{"a": "' + b64(jpeg(0))} + for file, text in shapes.items(): + with open(os.path.join(self.dir, file), "w") as f: + f.write(text) + api_logs.pack(self.dir) + for file, text in shapes.items(): + with open(os.path.join(self.dir, file)) as f: + self.assertEqual(f.read(), text, file) + self.assertIn(name, self.images()) + api_logs.restore(self.dir) + for file, text in shapes.items(): + with open(os.path.join(self.dir, file)) as f: + self.assertEqual(f.read(), text, file) + + def test_a_failed_write_leaves_every_file_whole_and_a_second_run_finishes(self): + replace = os.replace + for failing in ("images", "001_"): + with self.subTest(failing=failing): + def broken(src, dst): + if failing in dst: + raise OSError(28, "No space left on device") + return replace(src, dst) + + with mock.patch.object(api_logs.os, "replace", broken), self.assertRaises(OSError): + api_logs.pack(self.dir) + self.assertEqual(digest(self.dir), self.before) + self.assertEqual([n for n in os.listdir(os.path.join(self.dir, api_logs.IMAGES)) + if n.endswith(api_logs.TEMPORARY)], []) + done = api_logs.pack(self.dir) + self.assertEqual(done["packed"], 7) + api_logs.restore(self.dir) + self.assertEqual(digest(self.dir), self.before) + + def test_images_are_on_disk_before_a_file_names_them(self): + written = [] + write = api_logs._write + with mock.patch.object(api_logs, "_write", lambda path, *a: (written.append(path), write(path, *a))): + api_logs.pack(self.dir) + for i, path in enumerate(written): + if os.path.dirname(path) == self.dir: + with open(path, encoding="utf-8") as f: + named = {v.split("/")[-1] for v in json.dumps(json.load(f)).split('"') if v.startswith("images/")} + self.assertTrue(named <= {os.path.basename(p) for p in written[:i]}, path) + + def test_what_a_stopped_pack_left_half_written_is_removed(self): + api_logs.pack(self.dir) + left = [os.path.join(self.dir, ".001_x.json.abc" + api_logs.TEMPORARY), + os.path.join(self.dir, api_logs.IMAGES, ".a.jpg.abc" + api_logs.TEMPORARY)] + for path in left: + open(path, "w").close() + api_logs.pack(self.dir) + self.assertFalse(any(os.path.exists(p) for p in left)) + + def test_a_folder_another_process_works_on_is_left_alone(self): + fd = os.open(self.dir, os.O_RDONLY | os.O_DIRECTORY) + self.addCleanup(os.close, fd) + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time; fcntl.flock(int(sys.argv[1]), " + "fcntl.LOCK_EX); print(flush=True); time.sleep(30)", str(fd)], + pass_fds=(fd,), stdout=subprocess.PIPE) + self.addCleanup(lambda: (holder.kill(), holder.wait())) + holder.stdout.readline() + self.assertTrue(api_logs.pack(self.dir)["busy"]) + self.assertTrue(api_logs.restore(self.dir)["busy"]) + self.assertEqual(digest(self.dir), self.before) + + def test_a_missing_or_wrong_image_keeps_its_file_packed(self): + api_logs.pack(self.dir) + packed = digest(self.dir)["001_1700000000000.json"] + first = os.path.join(self.dir, api_logs.IMAGES, hashlib.sha256(jpeg(0)).hexdigest() + ".jpg") + os.remove(first) + self.assertEqual(api_logs.restore(self.dir), {"restored": 4, "failed": 3}) + self.assertEqual(digest(self.dir)["001_1700000000000.json"], packed) + with open(first, "wb") as f: + f.write(jpeg(99)) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 3}) + self.assertEqual(digest(self.dir)["001_1700000000000.json"], packed) + with open(first, "wb") as f: + f.write(jpeg(0)) + api_logs.restore(self.dir) + self.assertEqual(digest(self.dir), self.before) + + def test_links_are_not_followed(self): + outside = os.path.join(self.tmp, "outside") + os.makedirs(outside) + left = ".x" + api_logs.TEMPORARY + open(os.path.join(outside, left), "w").close() + os.symlink(outside, os.path.join(self.dir, api_logs.IMAGES)) + with self.assertRaises(OSError): + api_logs.pack(self.dir) + self.assertEqual(os.listdir(outside), [left]) + self.assertEqual(digest(self.dir), self.before) + os.remove(os.path.join(self.dir, api_logs.IMAGES)) + api_logs.pack(self.dir) + packed = digest(self.dir) + os.rename(os.path.join(self.dir, api_logs.IMAGES), os.path.join(self.tmp, "kept")) + for name in os.listdir(os.path.join(self.tmp, "kept")): + shutil.copy(os.path.join(self.tmp, "kept", name), outside) + os.symlink(outside, os.path.join(self.dir, api_logs.IMAGES)) + self.assertEqual(api_logs.restore(self.dir), {"restored": 0, "failed": 7}) + self.assertIn(left, os.listdir(outside)) + self.assertEqual(digest(self.dir), packed) + linked = os.path.join(self.tmp, "linked") + os.symlink(self.dir, linked) + with self.assertRaises(OSError): + api_logs.pack(linked) + + def test_the_command_line_packs_and_restores(self): + for act in ("pack", "restore"): + out = subprocess.run([sys.executable, os.path.join(HERE, "api_logs.py"), act, self.dir], + capture_output=True, text=True, check=True) + self.assertIn(self.dir, json.loads(out.stdout)) + self.assertEqual(digest(self.dir), self.before) + self.assertNotEqual(subprocess.run([sys.executable, os.path.join(HERE, "api_logs.py"), "unpack", self.dir], + capture_output=True).returncode, 0) + + +SCRIPT = """import json, os, sys, time +folder = os.path.join("results", "pyautogui", "screenshot", "m", "tasks", "001", "api_logs") +os.makedirs(folder, exist_ok=True) +for i, entry in enumerate(json.loads(sys.argv[1])): + with open(os.path.join(folder, "%03d_1.json" % (i + 1)), "w") as f: + json.dump(entry, f, ensure_ascii=False) + if i == 0 and sys.argv[2] == "stay": + open("ready", "w").close() + time.sleep(60) +""" + + +class Reaper(unittest.TestCase): + """The driver's reaper packs the unit's logs.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + env = mock.patch.dict(os.environ, NULLIUS_UPSTREAM=self.tmp) + env.start() + self.addCleanup(env.stop) + self.driver = module("osworld2_driver_logs", os.path.join(HERE, "driver.py"), NULLIUS_UPSTREAM=self.tmp) + self.folder = os.path.join(self.unit, "results", "pyautogui", "screenshot", "m", "tasks", "001", "api_logs") + self.addCleanup(self.driver.CLOSED.clear) + + def packed(self, n): + t0 = time.time() + while time.time() - t0 < LIMIT: + try: + files = [f for f in os.listdir(self.folder) if f.endswith(".json")] + if len(files) == n and all(read(os.path.join(self.folder, f)).startswith(api_logs.MARK) + for f in files): + return True + except OSError: + pass + time.sleep(0.2) + return False + + def run_script(self, stay, failed): + argv = [sys.executable, "-c", SCRIPT, json.dumps([call(i) for i in range(3)]), stay] + try: + self.driver._run(argv, self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run") + except RuntimeError as e: + failed.append(str(e)) + + def test_an_ended_run_is_packed(self): + failed = [] + self.run_script("go", failed) + self.assertEqual(failed, []) + self.assertTrue(self.packed(3)) + self.assertEqual(api_logs.restore(self.folder)["restored"], 3) + + def test_a_failed_run_is_packed(self): + argv = [sys.executable, "-c", SCRIPT + "sys.exit(3)\n", json.dumps([call(i) for i in range(2)]), "go"] + with self.assertRaises(RuntimeError): + self.driver._run(argv, self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run") + self.assertTrue(self.packed(2)) + + def test_a_stopped_run_is_packed_once_its_group_is_gone(self): + failed = [] + t = threading.Thread(target=self.run_script, args=("stay", failed), daemon=True) + t.start() + t0 = time.time() + while not os.path.exists(os.path.join(self.unit, "ready")) and time.time() - t0 < LIMIT: + time.sleep(0.05) + self.driver.CLOSED.set() + t.join(LIMIT) + self.assertTrue(failed and "was stopped" in failed[0]) + self.assertTrue(self.packed(1)) + + def test_a_group_still_there_is_not_packed(self): + packing = [] + with mock.patch.object(self.driver.os, "killpg"), mock.patch.object(self.driver, "_gone", lambda *_: False), \ + mock.patch.object(self.driver, "_pack", packing.append): + self.driver.reap(1, self.unit, False) + self.assertEqual(packing, []) + with mock.patch.object(self.driver.os, "killpg", side_effect=ProcessLookupError), \ + mock.patch.object(self.driver, "_pack", packing.append): + self.driver.reap(1, self.unit, False) + self.assertEqual(packing, [self.unit]) + + def test_no_packer_leaves_the_logs_and_the_unit_alone(self): + os.makedirs(self.folder) + with open(os.path.join(self.folder, "001_1.json"), "w") as f: + json.dump(call(0), f, ensure_ascii=False) + before = digest(self.folder) + with mock.patch.dict(sys.modules, api_logs=None): + self.driver._pack(self.unit) + self.assertEqual(digest(self.folder), before) + self.assertIn("api_logs not packed: ModuleNotFoundError", self.said()[0]) + + def said(self): + with open(os.path.join(self.unit, self.driver.PACK_LOG), encoding="utf-8") as f: + return f.read().splitlines() + + def test_each_folder_is_packed_on_its_own_and_said_in_one_line(self): + other = self.folder.replace("001", "002") + for folder in (self.folder, other): + os.makedirs(folder) + with open(os.path.join(folder, "001_1.json"), "w") as f: + json.dump(call(1), f, ensure_ascii=False) + before = digest(self.folder) + pack = api_logs.pack + + def broken(folder): + if folder == self.folder: + raise OSError(28, "No space left on device") + return pack(folder) + + with mock.patch.dict(sys.modules, api_logs=mock.Mock(pack=broken)): + self.driver._pack(self.unit) + self.assertEqual(digest(self.folder), before) + self.assertTrue(read(os.path.join(other, "001_1.json")).startswith(api_logs.MARK)) + lines = sorted(self.said(), key=lambda ln: "002" in ln) + self.assertEqual(len(lines), 2) + self.assertIn("001/api_logs {\"error\": \"OSError: [Errno 28] No space left on device\"}", lines[0]) + self.assertIn("002/api_logs {\"files\": 1, \"packed\": 1, \"images\": 2,", lines[1]) + self.assertRegex(lines[1], self.driver.RECORD) + + def test_a_unit_with_no_api_logs_says_nothing(self): + self.driver._pack(self.unit) + self.assertFalse(os.path.exists(os.path.join(self.unit, self.driver.PACK_LOG))) + + +class Snapshot(unittest.TestCase): + def test_a_hosts_snapshot_carries_the_packer(self): + from pathlib import Path + + from nullius.standard import payload + self.assertIn("api_logs.py", payload._entry(Path(HERE))) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_docker_create.py b/app/tests/test_osworld2_docker_create.py new file mode 100644 index 0000000..523f2e2 --- /dev/null +++ b/app/tests/test_osworld2_docker_create.py @@ -0,0 +1,610 @@ +"""OSWorld2's VM container create (startup/sitecustomize.py) with a busy Docker daemon: one container or none. +A fake docker-py, requests and docker command line stand in.""" +from __future__ import annotations + +import importlib.util +import ipaddress +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "nullius", "catalogue", + "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +PROVIDER = "happysixd/osworld-docker@sha256:" + "0" * 64 +NONE = {"NULLIUS_OPTION_WRITE_MBPS": "none", "NULLIUS_OPTION_WRITE_IOPS": "none"} +LIMITS = {"NULLIUS_OPTION_WRITE_MBPS": "10", "NULLIUS_OPTION_WRITE_IOPS": "100"} +DEVICE = "/dev/sdz" + + +class Timeout(Exception): + pass + + +class ReadTimeout(Timeout): + pass + + +class ConnectionError_(Exception): + pass + + +class ConnectTimeout(ConnectionError_, Timeout): + pass + + +class APIError(Exception): + pass + + +class Daemon: + """A fake daemon's create: slow (``seconds``), made ``late``, a slow ``inspect``, refused, or limits dropped (``drop``).""" + + def __init__(self, seconds=0, late=0, refuse=False, error=ReadTimeout, drop=False, inspect=0): + self.seconds, self.late, self.refuse, self.error, self.drop = seconds, late, refuse, error, drop + self.inspect = inspect + self.containers, self.creates, self.list_timeouts, self.clock = [], 0, [], 0.0 + self.pending, self.gave_up, self.info_timeouts = None, None, [] + + +class Container: + def __init__(self, daemon, image, kwargs): + self.daemon, self.image, self.kwargs, self.started, self.removed = daemon, image, kwargs, False, None + self.attrs = {} + + @property + def labels(self): + return self.kwargs.get("labels") or {} + + def reload(self): + kept = {} if self.daemon.drop else self.kwargs + self.attrs = {"HostConfig": {"BlkioDeviceWriteBps": kept.get("device_write_bps"), + "BlkioDeviceWriteIOps": kept.get("device_write_iops")}} + + def remove(self, **kwargs): + self.removed = kwargs + + def start(self): + self.started = True + + +class API: + def __init__(self, daemon, timeout=60): + self.daemon, self.timeout = daemon, timeout + + +class Client: + def __init__(self, daemon, timeout=60): + self.api = API(daemon, timeout) + + def info(self): + self.api.daemon.info_timeouts.append(self.api.timeout) + return {"DockerRootDir": "/var/lib/docker"} + + +class ContainerCollection: + def __init__(self, client): + self.client = client + + def create(self, image, command=None, **kwargs): + d = self.client.api.daemon + d.creates += 1 + if d.refuse: + raise APIError("refused") + if d.seconds > self.client.api.timeout: + d.clock = d.gave_up = self.client.api.timeout + if d.late is not None: + d.pending = Container(d, image, kwargs) + raise d.error("gave up") + d.containers.append(Container(d, image, kwargs)) + if d.inspect > self.client.api.timeout: + d.clock = d.gave_up = 2 * self.client.api.timeout + raise d.error("gave up") + return d.containers[-1] + + def list(self, all=False, filters=None): + d = self.client.api.daemon + d.list_timeouts.append(self.client.api.timeout) + if d.pending is not None and d.clock - d.gave_up >= d.late: + d.containers.append(d.pending) + d.pending = None + key, _, value = (filters or {}).get("label", "").partition("=") + return [c for c in d.containers if c.labels.get(key) == value] + + def run(self, image, command=None, **kwargs): + container = self.create(image=image, command=command, **kwargs) + container.start() + return container + + +class Patient(unittest.TestCase): + def setUp(self): + self.containers = types.ModuleType("docker.models.containers") + self.containers.ContainerCollection = ContainerCollection + self.original = ContainerCollection.create + self.addCleanup(setattr, ContainerCollection, "create", self.original) + requests = types.ModuleType("requests") + requests.exceptions = types.SimpleNamespace(Timeout=Timeout, ReadTimeout=ReadTimeout, + ConnectionError=ConnectionError_, ConnectTimeout=ConnectTimeout) + modules = mock.patch.dict(sys.modules, {"docker": types.ModuleType("docker"), + "docker.models": types.ModuleType("docker.models"), + "docker.models.containers": self.containers, + "requests": requests}) + modules.start() + self.addCleanup(modules.stop) + spec = importlib.util.spec_from_file_location("osworld2_startup_create", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + + def root_disk(root): + self.site._DISK[root] = DEVICE + return DEVICE + + disk = mock.patch.object(self.site, "root_disk", root_disk) + disk.start() + self.addCleanup(disk.stop) + self.swept = [] + + def sweeper(token): + r, w = os.pipe() + self.swept.append((token, r)) + self.addCleanup(os.close, r) + return w + + sweeper = mock.patch.object(self.site, "_sweeper", sweeper) + sweeper.start() + self.addCleanup(sweeper.stop) + + def said(self): + return [(token, os.read(r, 16)) for token, r in self.swept] + + def start(self, daemon, image=PROVIDER, timeout=60, environ=NONE, **kwargs): + ContainerCollection.create = self.original + self.site.wrap_docker_create(environ) + self.collection = ContainerCollection(Client(daemon, timeout)) + + def tick(seconds): + daemon.clock += seconds + + with mock.patch.object(self.site.time, "sleep", tick), \ + mock.patch.object(self.site.time, "monotonic", lambda: daemon.clock): + return self.collection.run(image, detach=True, **kwargs) + + def test_a_create_slower_than_docker_pys_60_s_goes_through(self): + daemon = Daemon(seconds=90) + with self.assertRaises(ReadTimeout): + ContainerCollection(Client(daemon)).run(PROVIDER) + made = self.start(Daemon(seconds=90)) + self.assertTrue(made.started) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + self.assertEqual(len(made.daemon.containers), 1) + + def test_a_create_the_client_timed_out_on_is_the_one_started(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING") as said: + made = self.start(daemon) + self.assertTrue(made.started) + self.assertEqual((daemon.creates, len(daemon.containers)), (1, 1)) + self.assertIs(made, daemon.containers[0]) + self.assertIn("ReadTimeout", said.output[0]) + + def test_a_request_answered_whose_inspect_timed_out_is_looked_for_and_started(self): + daemon = Daemon(inspect=150) + with self.assertLogs("nullius.osworld2", "WARNING"): + made = self.start(daemon, environ=LIMITS) + self.assertEqual((daemon.creates, len(daemon.containers)), (1, 1)) + self.assertIs(made, daemon.containers[0]) + self.assertEqual((made.started, made.removed), (True, None)) + self.assertEqual(daemon.list_timeouts, [self.site.LOOK_SECONDS]) + self.assertEqual(daemon.gave_up, 2 * self.site.CREATE_SECONDS) + + def test_each_look_waits_its_own_short_time_and_the_clients_comes_back(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + self.start(daemon) + self.assertGreater(len(daemon.list_timeouts), 1) + self.assertEqual(set(daemon.list_timeouts), {self.site.LOOK_SECONDS}) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + + def test_none_made_in_time_is_the_error_and_no_second_create(self): + for late in (None, 61): + with self.subTest(f"late {late}"): + daemon = Daemon(seconds=150, late=late) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.containers), (1, [])) + self.assertGreater(len(daemon.list_timeouts), 1) + self.assertLessEqual(daemon.clock - daemon.gave_up, self.site.ADOPT_SECONDS + self.site.POLL_SECONDS) + self.assertEqual(self.collection.client.api.timeout, self.site.CREATE_SECONDS) + + def test_a_failed_connection_stands_at_once(self): + for error in (ConnectionError_, ConnectTimeout): + with self.subTest(error.__name__): + daemon = Daemon(seconds=150, late=0, error=error) + with self.assertRaises(error): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.list_timeouts), (1, [])) + + def test_each_create_looks_only_for_its_own_label(self): + daemon = Daemon(seconds=150, late=None) + daemon.containers.append(Container(daemon, PROVIDER, {"labels": {self.site.LABEL: "another"}})) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertFalse(daemon.containers[0].started) + + def test_a_refusal_stands_at_once(self): + daemon = Daemon(refuse=True) + with self.assertRaises(APIError): + self.start(daemon) + self.assertEqual((daemon.creates, daemon.list_timeouts), (1, [])) + + def test_a_late_container_keeps_its_write_limit_and_is_started(self): + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + made = self.start(daemon, environ=LIMITS) + self.assertIs(made, daemon.containers[0]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteBps"], [{"Path": DEVICE, "Rate": 10 * 1024 * 1024}]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteIOps"], [{"Path": DEVICE, "Rate": 100}]) + self.assertEqual((made.started, made.removed), (True, None)) + + def test_a_late_container_without_its_write_limit_is_removed_and_fails_the_start(self): + daemon = Daemon(seconds=150, late=8, drop=True) + with self.assertLogs("nullius.osworld2", "WARNING"), self.assertRaises(self.site.WriteLimitError): + self.start(daemon, environ=LIMITS) + made = daemon.containers[0] + self.assertEqual((made.started, made.removed), (False, {"force": True, "v": True})) + + def test_dockers_root_is_asked_with_the_patient_client(self): + daemon = Daemon() + self.start(daemon, environ=LIMITS) + self.assertEqual(daemon.info_timeouts, [self.site.CREATE_SECONDS]) + + def test_another_image_is_left_alone(self): + daemon = Daemon() + made = self.start(daemon, "ubuntu:24.04") + self.assertEqual(made.kwargs, {"detach": True}) + self.assertEqual(self.collection.client.api.timeout, 60) + + def test_the_providers_ports_and_the_guests_a_unit_names_go_on_loopback_only(self): + given = {8006: 8006, 5000: 5001, 9222: 9223, 8080: 8081} + made = self.start(Daemon(), environ=dict(NONE, OSWORLD2_GUEST_PORTS="3000 3001"), ports=given) + self.assertEqual(made.kwargs["ports"], {8006: ("127.0.0.1", 8006), 5000: ("127.0.0.1", 5001), + 9222: ("127.0.0.1", 9223), 8080: ("127.0.0.1", 8081), + 3000: ("127.0.0.1", 3000), 3001: ("127.0.0.1", 3001)}) + self.assertEqual(given, {8006: 8006, 5000: 5001, 9222: 9223, 8080: 8081}) + self.assertEqual(self.start(Daemon(), ports=given).kwargs["ports"], + {k: ("127.0.0.1", v) for k, v in given.items()}) + self.assertEqual(self.start(Daemon(), ports={5000: None, 8006: ("127.0.0.1", 8006)}).kwargs["ports"], + {5000: ("127.0.0.1", None), 8006: ("127.0.0.1", 8006)}) + self.assertEqual(self.start(Daemon(), "ubuntu:24.04", environ=dict(NONE, OSWORLD2_GUEST_PORTS="3000")).kwargs, + {"detach": True}) + + def test_the_upstreams_retry_gets_a_second_vm_like_the_first(self): + environ = dict(LIMITS, OSWORLD2_GUEST_PORTS="3000") + daemon = Daemon() + first = self.start(daemon, environ=environ, ports={5000: 5000}) + first.remove() + daemon.seconds, daemon.late = 150, None + with mock.patch.object(self.site.time, "sleep", lambda s: setattr(daemon, "clock", daemon.clock + s)), \ + mock.patch.object(self.site.time, "monotonic", lambda: daemon.clock), self.assertRaises(ReadTimeout): + self.collection.run(PROVIDER, detach=True, ports={5000: 5001}) + self.assertTrue(first.removed.get("v")) + self.assertEqual([said for _, said in self.said()], [b"kept", b""]) + daemon.seconds = 0 + second = self.collection.run(PROVIDER, detach=True, ports={5000: 5001}) + self.assertEqual((daemon.creates, len(daemon.containers)), (3, 2)) + self.assertIsNot(second, first) + self.assertNotEqual(first.labels[self.site.LABEL], second.labels[self.site.LABEL]) + for made in (first, second): + self.assertEqual(made.kwargs["network"], self.site.NETWORK) + self.assertEqual(made.kwargs["ports"][3000], ("127.0.0.1", 3000)) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteBps"], + [{"Path": DEVICE, "Rate": 10 * 1024 * 1024}]) + self.assertEqual(made.attrs["HostConfig"]["BlkioDeviceWriteIOps"], [{"Path": DEVICE, "Rate": 100}]) + self.assertEqual((second.started, second.removed), (True, None)) + + def test_the_upstreams_removal_takes_the_vms_volume(self): + made = self.start(Daemon()) + made.remove() + self.assertEqual(made.removed, {"v": True}) + made.remove(force=True) + self.assertEqual(made.removed, {"force": True, "v": True}) + other = self.start(Daemon(), "ubuntu:24.04") + other.remove() + self.assertEqual(other.removed, {}) + + def test_only_a_create_never_kept_is_swept_for_later(self): + daemon = Daemon(seconds=150, late=self.site.ADOPT_SECONDS + 1) + with self.assertRaises(ReadTimeout): + self.start(daemon) + self.assertEqual(self.said(), [(daemon.pending.labels[self.site.LABEL], b"")]) + self.swept.clear() + with self.assertLogs("nullius.osworld2", "WARNING"): + self.start(Daemon(seconds=150, late=8)) + self.start(Daemon()) + for daemon in (Daemon(refuse=True), Daemon(seconds=150, error=ConnectionError_)): + with self.assertRaises(Exception): + self.start(daemon) + self.assertEqual([said for _, said in self.said()], [b"kept"] * 4) + + def test_a_create_cut_short_leaves_its_sweeper_to_the_process_end(self): + with self.assertRaises(KeyboardInterrupt): + self.start(Daemon(seconds=150, error=KeyboardInterrupt)) + (_, r), = self.swept + os.set_blocking(r, False) + with self.assertRaises(BlockingIOError): + os.read(r, 16) + + def test_a_longer_timeout_and_the_callers_labels_are_kept(self): + made = self.start(Daemon(), timeout=300, labels={"a": "1"}) + self.assertEqual(self.collection.client.api.timeout, 300) + self.assertEqual(made.labels["a"], "1") + self.assertRegex(made.labels[self.site.LABEL], r"^[0-9a-f]{32}$") + self.assertEqual(sorted(self.start(Daemon(), labels=["b"]).labels), ["b", self.site.LABEL]) + + def test_the_vm_goes_on_its_own_network_unless_the_create_names_one(self): + self.assertEqual(self.start(Daemon()).kwargs["network"], self.site.NETWORK) + self.assertEqual(self.start(Daemon(), network="another").kwargs["network"], "another") + self.assertNotIn("network", self.start(Daemon(), network_mode="host").kwargs) + daemon = Daemon(seconds=150, late=8) + with self.assertLogs("nullius.osworld2", "WARNING"): + self.assertEqual(self.start(daemon).kwargs["network"], self.site.NETWORK) + + def test_the_start_queue_waits_out_the_longest_first_start(self): + site = self.site + port_lists = site.PORT_LISTS * site.DOCKER_PY_SECONDS + root = request = inspect = reload = start = site.CREATE_SECONDS + late_search = site.ADOPT_SECONDS + site.POLL_SECONDS + 2 * site.LOOK_SECONDS + request_answered_inspect_timed_out = request + inspect + late_search + self.assertEqual(site.LONGEST_START, port_lists + root + request_answered_inspect_timed_out + reload + start) + spec = importlib.util.spec_from_file_location("osworld2_driver_create", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=STARTUP): + spec.loader.exec_module(driver) + self.assertGreaterEqual(driver.START_SECONDS, site.LONGEST_START) + + +SWEPT_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + ps) [ -e "$FAKE_MADE" ] && echo cid9 ;; + rm) [ -e "$FAKE_RM_FAILS" ] && exit 1 ;; +esac +exit 0 +""" + + +class Sweep(unittest.TestCase): + """sweep(), the abandoned create's later removal, with a fake docker command line.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(SWEPT_DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.log, self.made = os.path.join(self.tmp, "calls"), os.path.join(self.tmp, "made") + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.log, + FAKE_MADE=self.made, FAKE_RM_FAILS=os.path.join(self.tmp, "rm_fails")) + env.start() + self.addCleanup(env.stop) + spec = importlib.util.spec_from_file_location("osworld2_startup_sweep", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + + def calls(self): + try: + with open(self.log) as f: + return f.read().splitlines() + except OSError: + return [] + + def sweep(self, on_sleep=lambda n: None): + clock = [0.0] + + def sleep(seconds): + clock[0] += seconds + on_sleep(len(self.calls())) + + with mock.patch.object(self.site, "time", types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0])): + self.site.sweep("t" * 32) + return clock[0] + + def test_a_container_made_later_is_removed_with_its_volume(self): + self.sweep(lambda n: n >= 3 and open(self.made, "w").close()) + look = f"ps -a -q --filter label={self.site.LABEL}={'t' * 32}" + self.assertEqual(self.calls(), [look] * 4 + ["rm -f -v cid9"]) + + def test_a_refused_removal_is_tried_again(self): + open(self.made, "w").close() + rm_fails = os.environ["FAKE_RM_FAILS"] + open(rm_fails, "w").close() + self.sweep(lambda n: os.path.exists(rm_fails) and os.remove(rm_fails)) + self.assertEqual([c for c in self.calls() if c.startswith("rm ")], ["rm -f -v cid9"] * 2) + self.assertEqual(self.calls()[-1], "rm -f -v cid9") + + def test_none_made_ends_the_sweep_in_its_time(self): + took = self.sweep() + self.assertGreaterEqual(took, self.site.SWEEP_SECONDS) + self.assertEqual(len(self.calls()), self.site.SWEEP_SECONDS // self.site.LOOK_SECONDS) + self.assertTrue(all(c.startswith("ps ") for c in self.calls())) + + def test_the_sweeper_runs_apart_outlives_a_sigterm_and_sweeps_once_its_caller_is_gone(self): + for kept in (True, False): + with self.subTest(kept=kept): + self.swept_by_sweeper(kept) + + def swept_by_sweeper(self, kept): + token = ("k" if kept else "s") * 32 + for path in (self.made, self.log): + if os.path.exists(path): + os.remove(path) + w = self.site._sweeper(token) + + def sweepers(): + found = [] + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + if token.encode() in f.read(): + found.append(int(pid)) + except OSError: + pass + return found + + self.addCleanup(lambda: [os.kill(p, signal.SIGKILL) for p in sweepers()]) + found = sweepers() + self.assertEqual(len(found), 1) + self.assertEqual(os.getsid(found[0]), found[0]) + t0 = time.time() + while not self.ignores_sigterm(found[0]) and time.time() - t0 < 20: + time.sleep(0.05) + os.kill(found[0], signal.SIGTERM) + open(self.made, "w").close() + if kept: + self.site._kept(w) + else: + os.close(w) + while sweepers() and time.time() - t0 < 30: + time.sleep(0.1) + self.assertEqual(sweepers(), []) + self.assertEqual(self.calls()[-1:], [] if kept else ["rm -f -v cid9"]) + + @staticmethod + def ignores_sigterm(pid): + with open(f"/proc/{pid}/status") as f: + ignored = next(ln.split()[1] for ln in f if ln.startswith("SigIgn:")) + return bool(int(ignored, 16) & 1 << (signal.SIGTERM - 1)) + + +PROVIDER_PY = "LOCK_TIMEOUT = 10\n" + + +class StartQueue(unittest.TestCase): + """The provider's port lock, with a fake desktop_env and the startup folder first on PYTHONPATH.""" + + def test_every_vm_start_waits_for_the_providers_port_lock(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + package = os.path.join(tmp, "desktop_env", "providers", "docker") + os.makedirs(package) + for folder in ("desktop_env", os.path.join("desktop_env", "providers"), package): + open(os.path.join(tmp, folder, "__init__.py"), "w").close() + with open(os.path.join(package, "provider.py"), "w") as f: + f.write(PROVIDER_PY) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + for option, wanted in (("docker", "-1"), ("aws", "10")): + with self.subTest(option=option): + done = subprocess.run([sys.executable, "-c", "import desktop_env.providers.docker.provider as m; " + "print(m.LOCK_TIMEOUT)"], cwd=tmp, capture_output=True, text=True, timeout=60, + env=dict(environ, PYTHONPATH=os.pathsep.join([STARTUP, tmp]), + NULLIUS_OPTION_VM_PROVIDER=option)) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertEqual(done.stdout.strip(), wanted) + + +class Said: + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + +class Network(unittest.TestCase): + """network(), with a fake docker command line.""" + + def setUp(self): + spec = importlib.util.spec_from_file_location("osworld2_startup_network", os.path.join(STARTUP, "sitecustomize.py")) + self.site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.site) + self.networks, self.calls, self.refuse = {}, [], None + self.shape = f"1|{self.site.SUBNET} {self.site.GATEWAY} " + + def docker(self, *args): + self.calls.append(args) + name = args[-1] + if args[:2] == ("network", "inspect"): + return Said(0, self.networks[name] + "\n") if name in self.networks else Said(1, "", f"Error: no such network: {name}") + if args[:2] == ("network", "create"): + if self.refuse: + return Said(1, "", self.refuse) + if name in self.networks: + return Said(1, "", f"Error response from daemon: network with name {name} already exists") + subnet, gateway = args[args.index("--subnet") + 1], args[args.index("--gateway") + 1] + label = "1" if f"{self.site.NETWORK_LABEL}=1" in args else "" + self.networks[name] = f"{label}|{subnet} {gateway} " + return Said(0, "id") + raise AssertionError(args) + + def test_a_missing_network_is_made_with_its_subnet_gateway_and_label(self): + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + create, = [c for c in self.calls if c[1] == "create"] + for flag in (("--driver", "bridge"), ("--subnet", self.site.SUBNET), ("--gateway", self.site.GATEWAY), + ("--label", f"{self.site.NETWORK_LABEL}=1")): + self.assertIn(" ".join(flag), " ".join(create)) + self.calls.clear() + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + self.assertEqual([c[1] for c in self.calls], ["inspect"]) + + def test_another_maker_at_the_same_moment_is_no_error(self): + first = self.docker + + def racing(*args): + if args[:2] == ("network", "create"): + first("network", "create", "--subnet", self.site.SUBNET, "--gateway", self.site.GATEWAY, + "--label", f"{self.site.NETWORK_LABEL}=1", args[-1]) + return first(*args) + + self.assertEqual(self.site.network(racing), self.site.NETWORK) + self.assertEqual(self.networks, {self.site.NETWORK: self.shape}) + + def test_a_network_of_that_name_on_another_subnet_is_refused(self): + self.networks[self.site.NETWORK] = "1|172.17.0.0/16 172.17.0.1 " + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertIn(f"docker network rm {self.site.NETWORK}", str(raised.exception)) + self.assertFalse([c for c in self.calls if c[1] == "create"]) + + def test_another_owner_s_network_of_that_name_is_never_offered_for_removal(self): + self.networks[self.site.NETWORK] = "|172.17.0.0/16 172.17.0.1 " + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertNotIn("docker network rm", str(raised.exception)) + self.assertIn("Nullius did not make it", str(raised.exception)) + self.assertIn(f"{self.site.NETWORK} and {self.site.SUBNET} are free", str(raised.exception)) + + def test_a_network_made_by_hand_with_that_subnet_and_gateway_is_used(self): + self.networks[self.site.NETWORK] = f"|{self.site.SUBNET} {self.site.GATEWAY} " + self.assertEqual(self.site.network(self.docker), self.site.NETWORK) + self.assertFalse([c for c in self.calls if c[1] == "create"]) + + def test_a_network_that_cannot_be_made_says_why(self): + self.refuse = "Error response from daemon: Pool overlaps with other one on this address space" + with self.assertRaises(self.site.NetworkError) as raised: + self.site.network(self.docker) + self.assertIn("Pool overlaps", str(raised.exception)) + self.assertIn(self.site.SUBNET, str(raised.exception)) + + def test_the_label_is_read_with_the_subnet(self): + self.site.network(self.docker) + inspect = self.calls[0] + self.assertIn('{{index .Labels "' + self.site.NETWORK_LABEL + '"}}|', inspect[inspect.index("-f") + 1]) + + def test_the_subnet_is_none_docker_takes_for_its_own_bridges(self): + subnet = ipaddress.ip_network(self.site.SUBNET) + for taken in ("172.16.0.0/12", "192.168.0.0/16"): + self.assertFalse(subnet.overlaps(ipaddress.ip_network(taken)), taken) + self.assertIn(ipaddress.ip_address(self.site.GATEWAY), subnet) + self.assertTrue(subnet.is_private) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_driver.py b/app/tests/test_osworld2_driver.py new file mode 100644 index 0000000..42e4452 --- /dev/null +++ b/app/tests/test_osworld2_driver.py @@ -0,0 +1,407 @@ +"""OSWorld2's driver cleans up after a unit whose docker hangs or whose vm/volumes note cannot be trusted. +A fake ``docker`` on PATH stands in.""" +from __future__ import annotations + +import errno +import importlib.util +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +VOLUME = "a" * 64 +OTHER = "b" * 64 +HANG = "30" +LIMIT = 10 +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +[ -z "$FAKE_ALIVE" ] || [ -e "$FAKE_ALIVE" ] || exit 0 +c=cid1 v="$FAKE_VOLUME" +[ -n "$FAKE_SECOND" ] && [ -e "$FAKE_SECOND" ] && c=cid2 v="$FAKE_OTHER" +case "$1" in + ps) echo "$c" ;; + inspect) i=0 + while [ "$i" -lt "$(( ${FAKE_DOCKER_DELAY:-0} * 10 ))" ] && [ ! -e "$FAKE_RELEASE" ]; do sleep 0.1; i=$((i + 1)); done + printf '%s\\t[{"Type":"bind","Source":"%s/vm/x.qcow2"},{"Type":"volume","Name":"%s","Source":"/v"}]\\n' "$c" "$FAKE_UNIT" "$v" ;; +esac +""" + + +def load(up): + spec = importlib.util.spec_from_file_location("osworld2_driver", os.path.join(HERE, "driver.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(module) + return module + + +CHILD = """import os, signal, sys, time +def term(*_): + with open(sys.argv[1], "w") as f: + f.write(repr(time.time())) + if len(sys.argv) > 3: + os.remove(sys.argv[3]) + sys.exit(0) +signal.signal(signal.SIGTERM, term) +open(sys.argv[2], "w").close() +time.sleep(60) +""" + +KILLED = """import importlib.util, sys +here, unit, started = sys.argv[1:4] +sys.argv = ["driver.py"] +sys.path.insert(0, here) +spec = importlib.util.spec_from_file_location("osworld2_driver", here + "/driver.py") +driver = importlib.util.module_from_spec(spec) +spec.loader.exec_module(driver) +driver.START_LOCK = unit + "/start.lck" +driver._run(["sh", "-c", "echo '%s'; echo $$ > %s; sleep 60" % (driver.STARTED, started)], unit, None, + unit + "/nop.log", "the run", stagger=True) +""" + + +class Cleanup(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(os.path.join(self.unit, "vm")) + self.note = os.path.join(self.unit, "vm", "volumes") + self.calls = os.path.join(self.tmp, "calls") + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_UNIT=self.unit, FAKE_VOLUME=VOLUME, FAKE_DOCKER_DELAY="0", + FAKE_RELEASE=os.path.join(self.tmp, "release")) + env.start() + self.addCleanup(env.stop) + threads = set(threading.enumerate()) + self.addCleanup(lambda: [t.join(30) for t in set(threading.enumerate()) - threads]) + self.addCleanup(lambda: open(os.environ["FAKE_RELEASE"], "w").close()) + self.driver = load(self.tmp) + self.driver.START_LOCK = os.path.join(self.tmp, "start.lck") + + def lines(self): + try: + with open(self.calls) as f: + return f.read().splitlines() + except OSError: + return [] + + def removed(self): + return [ln for ln in self.lines() if ln.startswith(("rm ", "volume rm "))] + + def remove(self): + t = threading.Thread(target=self.driver._remove_vms, args=(self.unit,), daemon=True) + t.start() + t.join(10) + self.assertFalse(t.is_alive(), "the removal waited") + return self.removed() + + def run_unit(self, failed, command="sleep 30"): + """The driver's run of a process that starts a VM, in a thread; every process it starts is waited for.""" + started = [] + + def popen(*a, **kw): + p = subprocess.Popen(*a, **kw) + started.append(p) + threading.Thread(target=p.wait, daemon=True).start() + return p + + self.driver.subprocess = types.SimpleNamespace(**{k: getattr(subprocess, k) for k in dir(subprocess) + if not k.startswith("__") and k != "Popen"}, Popen=popen) + + def end(): + for p in started: + if p.poll() is None: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + + self.addCleanup(end) + + def go(): + try: + self.driver._run(["sh", "-c", f"echo '{self.driver.STARTED}'; {command}"], self.unit, dict(os.environ), + os.path.join(self.unit, "nop.log"), "the run", stagger=True) + except RuntimeError as e: + failed.append(str(e)) + + t = threading.Thread(target=go) + t.start() + return t + + def test_another_user_s_start_lock_serves_too(self): + open(self.driver.START_LOCK, "w").close() + os.chmod(self.driver.START_LOCK, 0o444) + self.driver._started = self.driver._reaper = self.driver._remove_vms = lambda *a: None + self.driver._run(["true"], self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), "the run", stagger=True) + + def test_a_slow_docker_does_not_hold_the_stop(self): + os.environ["FAKE_DOCKER_DELAY"] = HANG + reaped, failed = [], [] + self.driver._reaper = lambda pgid, *_: reaped.append(time.time()) + t = self.run_unit(failed) + time.sleep(2.5) + closed = time.time() + self.driver.CLOSED.set() + t.join(10) + self.assertTrue(reaped and failed) + self.assertLess(reaped[0] - closed, LIMIT) + + def test_a_stopped_unit_hands_the_reaper_the_volumes_it_saw(self): + alive = os.path.join(self.tmp, "alive") + open(alive, "w").close() + os.makedirs(self.note) + env = mock.patch.dict(os.environ, FAKE_ALIVE=alive, NULLIUS_UPSTREAM=self.tmp) + env.start() + self.addCleanup(env.stop) + failed = [] + t = self.run_unit(failed) + t0 = time.time() + while not any(ln.startswith("inspect") for ln in self.lines()) and time.time() - t0 < LIMIT: + time.sleep(0.05) + time.sleep(0.5) + os.remove(alive) + self.driver.CLOSED.set() + t.join(10) + self.assertTrue(failed) + t0 = time.time() + while f"volume rm {VOLUME}" not in self.removed() and time.time() - t0 < LIMIT: + time.sleep(0.1) + self.assertEqual(self.removed(), [f"volume rm {VOLUME}"]) + + def reapers(self): + found = [] + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + argv = f.read().split(b"\0") + except OSError: + continue + if b"--reap" in argv and self.unit.encode() in argv: + found.append(int(pid)) + return found + + @staticmethod + def ignores_sigterm(pid): + with open(f"/proc/{pid}/status") as f: + ignored = next(ln.split()[1] for ln in f if ln.startswith("SigIgn:")) + return bool(int(ignored, 16) & 1 << (signal.SIGTERM - 1)) + + def test_a_driver_killed_mid_run_leaves_its_reaper_the_vm_and_its_volume(self): + os.environ["NULLIUS_UPSTREAM"] = self.tmp + started = os.path.join(self.tmp, "started") + self.addCleanup(lambda: [os.kill(p, signal.SIGKILL) for p in self.reapers()]) + for sig in (signal.SIGTERM, signal.SIGKILL): + with self.subTest(signal=sig.name): + for path in (started, self.calls): + if os.path.exists(path): + os.remove(path) + driver = subprocess.Popen([sys.executable, "-c", KILLED, HERE, self.unit, started], + start_new_session=True) + self.addCleanup(lambda d=driver: (d.poll() is None and os.killpg(d.pid, signal.SIGKILL), d.wait())) + t0 = time.time() + while not (os.path.exists(started) and os.path.getsize(started)) and time.time() - t0 < LIMIT: + time.sleep(0.05) + with open(started) as f: + vm = int(f.read()) + while not (self.reapers() and self.ignores_sigterm(self.reapers()[0])) and time.time() - t0 < LIMIT: + time.sleep(0.05) + self.assertEqual(len(self.reapers()), 1) + os.killpg(driver.pid, sig) + driver.wait(LIMIT) + for pid in self.reapers(): + os.kill(pid, signal.SIGTERM) + t0 = time.time() + while f"volume rm {VOLUME}" not in self.removed() and time.time() - t0 < 3 * LIMIT: + time.sleep(0.1) + self.assertEqual(self.removed()[:2], ["rm -f -v cid1", f"volume rm {VOLUME}"]) + with self.assertRaises(ProcessLookupError): + os.killpg(vm, 0) + for pid in self.reapers(): + os.kill(pid, signal.SIGKILL) + + def test_a_reaper_that_cannot_start_ends_the_script_and_its_vm_at_once(self): + reaped = [] + with mock.patch.object(self.driver, "_arm", mock.Mock(side_effect=OSError(11, "Resource temporarily unavailable"))), \ + mock.patch.object(self.driver, "reap", lambda *a: reaped.append(a)), \ + self.assertRaises(self.driver.EnvironmentSetupError): + self.driver._run(["sh", "-c", "sleep 30"], self.unit, dict(os.environ), os.path.join(self.unit, "nop.log"), + "the run", stagger=True) + self.assertEqual(len(reaped), 1) + self.assertEqual(reaped[0][1:], (self.unit, True)) + os.killpg(reaped[0][0], signal.SIGKILL) + + def test_a_run_cut_short_by_an_error_still_lets_its_reaper_go(self): + released = [] + self.driver._reaper = lambda pgid, *a: (released.append(pgid), self.driver._ARMED.pop(pgid)) + self.driver._arm = lambda pgid, *a: self.driver._ARMED.__setitem__(pgid, -1) + broken = types.SimpleNamespace(Thread=mock.Mock(side_effect=RuntimeError("can't start new thread")), + Event=threading.Event) + with mock.patch.object(self.driver, "threading", broken), self.assertRaises(RuntimeError): + self.driver._run(["sh", "-c", f"echo '{self.driver.STARTED}'; sleep 30"], self.unit, dict(os.environ), + os.path.join(self.unit, "nop.log"), "the run", stagger=True) + self.assertEqual(len(released), 1) + self.assertEqual(self.driver._ARMED, {}) + os.killpg(released[0], signal.SIGKILL) + + def test_a_retried_setup_leaves_neither_vm_nor_either_disk(self): + second, go = os.path.join(self.tmp, "second"), os.path.join(self.tmp, "go") + env = mock.patch.dict(os.environ, FAKE_SECOND=second, FAKE_OTHER=OTHER) + env.start() + self.addCleanup(env.stop) + self.driver.NOTE_SECONDS = 0.1 + self.driver._reaper = lambda *a: None + failed = [] + + def noted(name): + t0 = time.time() + while time.time() - t0 < LIMIT: + try: + with open(self.note) as f: + if name in f.read().split(): + return True + except OSError: + pass + time.sleep(0.05) + return False + + t = self.run_unit(failed, f"while [ ! -e {go} ]; do sleep 0.1; done") + self.assertTrue(noted(VOLUME)) + open(second, "w").close() + self.assertTrue(noted(OTHER)) + open(go, "w").close() + t.join(LIMIT) + self.assertFalse(t.is_alive()) + self.assertEqual(failed, []) + self.assertEqual(self.removed(), ["rm -f -v cid2", f"volume rm {VOLUME} {OTHER}"]) + + def test_the_volumes_seen_are_noted(self): + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + with open(self.note) as f: + self.assertEqual(f.read().split(), [VOLUME]) + + def test_a_name_in_the_note_is_removed(self): + with open(self.note, "w") as f: + f.write(f"someones-volume\n{OTHER}\n{OTHER}\n../x\n{VOLUME}x\n") + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {OTHER} {VOLUME}"]) + + def test_the_note_is_read_to_its_limit(self): + with open(self.note, "w") as f: + f.write(" " * self.driver.NOTE_BYTES + OTHER + "\n") + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def test_a_note_not_the_units_own_is_not_evidence(self): + elsewhere = os.path.join(self.tmp, "elsewhere") + with open(elsewhere, "w") as f: + f.write(OTHER + "\n") + for make in (lambda: os.symlink(elsewhere, self.note), lambda: os.link(elsewhere, self.note), + lambda: os.makedirs(self.note), lambda: os.mkfifo(self.note)): + with self.subTest(make=make): + make() + self.assertEqual(self.remove()[-2:], ["rm -f -v cid1", f"volume rm {VOLUME}"]) + if os.path.isdir(self.note) and not os.path.islink(self.note): + os.rmdir(self.note) + else: + os.remove(self.note) + with open(elsewhere) as f: + self.assertEqual(f.read(), OTHER + "\n") + + def test_a_full_disk_still_removes(self): + write = os.write + + def full(fd, data): + if os.readlink(f"/proc/self/fd/{fd}") == self.note: + raise OSError(errno.ENOSPC, "No space left on device") + return write(fd, data) + + with mock.patch.object(self.driver.os, "write", full): + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def test_no_vm_folder_still_removes(self): + os.rmdir(os.path.join(self.unit, "vm")) + self.assertEqual(self.remove(), ["rm -f -v cid1", f"volume rm {VOLUME}"]) + + def child(self, *alive): + """A process group of its own whose SIGTERM is timed, and can remove a container (FAKE_ALIVE).""" + signalled, ready = os.path.join(self.tmp, "signalled"), os.path.join(self.tmp, "ready") + for path in (signalled, ready): + if os.path.exists(path): + os.remove(path) + p = subprocess.Popen([sys.executable, "-c", CHILD, signalled, ready, *alive], start_new_session=True) + threading.Thread(target=p.wait, daemon=True).start() + self.addCleanup(lambda: (p.poll() is None and os.killpg(p.pid, signal.SIGKILL), p.wait())) + t0 = time.time() + while not os.path.exists(ready) and time.time() - t0 < 10: + time.sleep(0.05) + return p, signalled + + def test_a_hung_docker_does_not_hold_the_reapers_signal(self): + os.environ["FAKE_DOCKER_DELAY"] = HANG + p, signalled = self.child() + self.driver.SNAPSHOT_SECONDS = 1 + self.driver.SWEEPS = () + t0 = time.time() + self.driver.reap(p.pid, self.unit, True) + p.wait(10) + with open(signalled) as f: + self.assertLess(float(f.read()) - t0, LIMIT) + + def test_a_volume_seen_before_the_signal_is_removed_when_the_note_cannot_keep_it(self): + write = os.write + + def full(fd, data): + if os.readlink(f"/proc/self/fd/{fd}") == self.note: + raise OSError(errno.ENOSPC, "No space left on device") + return write(fd, data) + + alive = os.path.join(self.tmp, "alive") + os.environ["FAKE_ALIVE"] = alive + self.driver.SWEEPS = (0,) + for name, keep in (("refused", lambda: os.makedirs(self.note)), ("full", lambda: mock.patch.object(self.driver.os, "write", full))): + with self.subTest(note=name): + open(alive, "w").close() + open(self.calls, "w").close() + patch = keep() + if patch is not None: + patch.start() + p, _ = self.child(alive) + try: + self.driver.reap(p.pid, self.unit, True) + finally: + if patch is not None: + patch.stop() + p.wait(10) + self.assertFalse(os.path.exists(alive)) + self.assertEqual(self.removed(), [f"volume rm {VOLUME}"]) + if os.path.isdir(self.note): + os.rmdir(self.note) + + def test_the_reaper_notes_before_its_signal(self): + seen = [] + + def killpg(pgid, sig): + with open(self.note) as f: + seen.append(f.read()) + raise OSError(errno.ESRCH, "No such process") + + with mock.patch.object(self.driver.os, "killpg", killpg), mock.patch.object(self.driver, "SWEEPS", ()): + self.driver.reap(1, self.unit, True) + self.assertEqual(seen, [VOLUME + "\n"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_evaluation.py b/app/tests/test_osworld2_evaluation.py new file mode 100644 index 0000000..e00ba51 --- /dev/null +++ b/app/tests/test_osworld2_evaluation.py @@ -0,0 +1,403 @@ +"""OSWorld2's failures by whose they are: an evaluation that could not read the VM or a site is the score's +error, never a 0; the environment's is a setup error; the upstream's words never reach the message.""" +from __future__ import annotations + +import contextlib +import fcntl +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +from tests.test_osworld2_judge import load_driver + +STEP = "[2026-11-10 22:09:08,778 INFO lib_run_single/1579-EnvProcess-1] Done: False" +RESULT = "[2026-11-10 22:09:38,788 INFO lib_run_single/1622-EnvProcess-1] Result: 0.00" +SITE = ("[2026-11-10 22:09:38,787 ERROR state/82-EnvProcess-1] [GET_STATE_WITH_COOKIE] Failed to fetch state from " + "http://site.example.nip.io/api/state: HTTPConnectionPool(host='site.example.nip.io', port=80): ") +FILE = "[2026-11-10 22:09:20,000 {} desktopenv.pycontroller] {}" + + +def tried(status=None): + return [FILE.format("ERROR", f"Failed to get file. Status code: {status}" if status else + "An error occurred while trying to get the file: ('Connection broken: IncompleteRead(0 bytes read)')"), + FILE.format("INFO", "Retrying to get file.")] + + +class Evaluation(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.tmp = os.path.join(trial, "workspace") + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "081") + os.makedirs(self.where) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("0.0\n") + + def score(self, *lines): + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in lines)) + return self.driver.score("tasks__081", self.tmp, setup="original") + + def unread(self, *lines): + with self.assertRaises(self.driver.VerifierError) as raised: + self.score(*lines) + return str(raised.exception) + + def test_every_check_of_the_evaluation_is_the_scores_error_by_its_class(self): + self.assertIn("the LLM grader failed in compare_text_with_llm", str(self.unread( + STEP, "[2026-11-10 22:09:30,000 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: x", RESULT))) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("2.0\n") + self.assertIn("not a score from 0 to 1", self.unread(STEP, RESULT)) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("") + self.assertEqual("result.txt could not be read as a number", self.unread(STEP, RESULT)) + os.remove(os.path.join(self.where, "result.txt")) + self.assertIn("no result.txt", self.unread(STEP, RESULT)) + + def test_setup_errors_the_last_setup_went_on_from_are_counted_beside_the_score(self): + at = "[2026-11-10 22:00:0{} {} {}] {}" + reset = at.format(1, "INFO", "desktop_env/330-EnvProcess-1", "Environment has been used, reverting to snapshot x...") + clean = at.format(1, "INFO", "desktopenv.env", "Environment is clean, skipping snapshot revert (provider: docker).") + failed = at.format(2, "ERROR", "setup/467-EnvProcess-1", "Failed to launch application. Status code: x") + logged = at.format(2, "ERROR", "desktopenv.setup", "An error occurred while trying to send the request: x") + retried = [at.format(2, "ERROR", "setup/686-EnvProcess-1", "Attempt 1: Failed to connect, retrying. Error: x"), + at.format(2, "ERROR", "setup/311-EnvProcess-1", "Upload attempt 1 failed for x: y"), + at.format(2, "ERROR", "desktopenv.setup", "Failed to download x caused by y. Retrying... (2 attempts left)")] + self.assertNotIn("setup_errors", self.score(reset, *retried, STEP, RESULT)) + self.assertNotIn("setup_errors", self.score(failed, failed, reset, STEP, RESULT)) + self.assertEqual(self.score(reset, failed, *retried, logged, STEP, RESULT)["setup_errors"], 2) + self.assertEqual(self.score(failed, clean, "\x1b[31m" + logged + "\x1b[0m", STEP, RESULT)["setup_errors"], 1) + muse = "[2026-11-10 22:09:08,778 INFO desktopenv.experiment/1579-EnvProcess-1] Step 3: x" + self.assertEqual(self.score(reset, failed, STEP, logged, STEP, failed, logged, RESULT)["setup_errors"], 2) + self.assertNotIn("setup_errors", self.score(reset, muse, failed, RESULT)) + self.assertNotIn("setup_errors", self.score(reset, at.format(2, "ERROR", "setup/306-EnvProcess-1", + "Failed to upload file x. Status code: 500"), STEP)) + self.assertEqual(self.score(reset, at.format(2, "ERROR", "setup/1031-EnvProcess-1", + "Failed to upload file. Status code: 500"), STEP)["setup_errors"], 1) + done = at.format(3, "INFO", "desktopenv.env", "Environment setup complete.") + self.assertEqual(self.score(reset, failed, done, logged, RESULT)["setup_errors"], 1) + self.assertEqual(self.score(reset, failed, done, logged, STEP, RESULT)["setup_errors"], 2) + + def test_a_sites_state_that_timed_out_is_the_scores_error_with_the_upstreams_value(self): + said = self.unread(STEP, SITE + "Read timed out. (read timeout=30)", RESULT) + self.assertEqual(said, "the evaluation could not read a site's state (timed out), and the upstream scored the " + "task 0.0 without it; see the unit's eval.log") + self.assertIn("(no answer)", self.unread(STEP, SITE + "Max retries exceeded", RESULT)) + self.assertIn("a site's file (HTTP 502)", self.unread( + STEP, SITE.replace("fetch state", "download file") + "502 Server Error: Bad Gateway", RESULT)) + + def test_a_site_that_answered_leaves_the_upstreams_score(self): + for said in ("404 Client Error: NOT FOUND for url", "500 Server Error: INTERNAL SERVER ERROR for url"): + self.assertEqual(self.score(STEP, SITE + said, RESULT)["reward"], 0.0) + + def test_a_sites_read_that_failed_stands_though_a_later_read_succeeds(self): + read = "[2026-11-10 22:09:39,000 INFO state/96-EnvProcess-1] [GET_STATE_WITH_COOKIE] Returning state JSON directly" + self.assertIn("a site's state (timed out)", self.unread(STEP, SITE + "Read timed out.", read, RESULT)) + + def test_a_vm_read_that_gave_up_on_a_broken_connection_is_the_scores_error(self): + give_up = FILE.format("ERROR", "Failed to get file.") + self.assertIn("the VM's file (a broken connection)", self.unread(STEP, *tried(404) * 2, *tried(), give_up, RESULT)) + self.assertEqual(self.score(STEP, *tried() * 2, *tried(500), give_up, RESULT)["reward"], 0.0) + + def test_an_absent_file_or_a_recovered_read_is_no_failure(self): + give_up = FILE.format("ERROR", "Failed to get file.") + self.assertEqual(self.score(STEP, *tried(404) * 3, give_up, RESULT)["reward"], 0.0) + self.assertEqual(self.score(STEP, *tried(), FILE.format("INFO", "File downloaded successfully"), + "[2026-11-10 22:09:30,000 ERROR desktopenv.task081] Failed to get file.", + RESULT)["reward"], 0.0) + + def test_other_loggers_words_neither_fail_it_nor_hide_a_failure(self): + task = "[2026-11-10 22:09:21,000 {} desktopenv.task081] {}" + self.assertEqual(self.score(STEP, task.format("ERROR", SITE[SITE.index("] ") + 2:] + "Read timed out."), + task.format("ERROR", "An error occurred while trying to get the words: x"), + task.format("ERROR", "Failed to get the words."), + task.format("INFO", "Result: 1.00"), RESULT)["reward"], 0.0) + said = self.unread(STEP, *tried(), task.format("INFO", "still waiting"), " a continued line", + FILE.format("ERROR", "Failed to get file."), RESULT) + self.assertIn("the VM's file (a broken connection)", said) + self.assertNotIn("words", said) + + def test_only_the_evaluation_after_the_last_step_counts(self): + self.assertEqual(self.score(SITE + "Read timed out.", STEP, RESULT)["reward"], 0.0) + self.assertEqual(self.score(STEP, RESULT, SITE + "Read timed out.")["reward"], 0.0) + self.assertEqual(self.score(SITE + "Read timed out.", RESULT)["reward"], 0.0) + + def test_every_phase_of_the_last_multi_phase_episode_counts(self): + mark = "[2026-11-10 22:09:{:02},000 INFO lib_run_single/{}-EnvProcess-1] {}" + step, start = (lambda n: mark.format(n, 776, f"Phase {n} Step 1: x")), mark.format(30, 712, "Starting phase 2 in trajectory 2") + end = mark.format(40, 845, "Multi-phase result: 0.5000") + fail, lost = SITE + "Read timed out.", [*tried(), FILE.format("ERROR", "Failed to get file.")] + self.assertIn("a site's state (timed out)", self.unread(step(1), fail, start, step(2), end)) + self.assertIn("a site's state (timed out)", self.unread(step(1), start, step(2), fail, end)) + self.assertIn("the VM's file (a broken connection)", self.unread(step(1), *lost, start, step(2), end)) + self.assertEqual(self.score(step(1), start, fail, step(2), end)["reward"], 0.0) + self.assertEqual(self.score(step(1), start, fail, end)["reward"], 0.0) + self.assertEqual(self.score(step(1), fail, end, step(1), end)["reward"], 0.0) + self.assertEqual(self.score(step(1), fail, start, step(1), start, step(2), end)["reward"], 0.0) + asked = mark.format(2, 750, "Phase 1 User simulator Q: x | A: y") + self.assertEqual(self.score(step(1), fail, start, asked, start, step(2), end)["reward"], 0.0) + self.assertIn("a site's state (timed out)", self.unread(asked, fail, end)) + + def test_a_task_that_retries_its_site_reads_keeps_its_score_but_not_a_lost_vm_read(self): + mark = "[2026-11-10 22:09:{:02},000 INFO lib_run_single/{}-EnvProcess-1] {}" + step, end = mark.format(1, 776, "Phase 1 Step 1: x"), mark.format(40, 845, "Multi-phase result: 0.5000") + self.where = self.where.replace("081", "069") + os.makedirs(self.where) + with open(os.path.join(self.where, "result.txt"), "w") as f: + f.write("0.5\n") + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in (step, SITE + "Read timed out.", end))) + self.assertEqual(self.driver.score("tasks__069", self.tmp, setup="original")["reward"], 0.5) + with open(os.path.join(self.where, "eval.log"), "w") as f: + f.write("".join(ln + "\n" for ln in (step, *tried(), FILE.format("ERROR", "Failed to get file."), end))) + with self.assertRaises(RuntimeError): + self.driver.score("tasks__069", self.tmp, setup="original") + + def test_the_null_controls_unread_evaluation_is_its_error(self): + def run(argv, cwd, env, log, *a, **kw): + with open(os.path.join(argv[-1], "result.txt"), "w") as f: + f.write("0.0\n") + with open(log, "w") as f: + f.write("\n".join(["[2026-11-10 22:09:08,778 INFO desktopenv.experiment] Done: True", + SITE.replace("state/82-EnvProcess-1", "desktopenv.getters.state") + "Read timed out.", + "[2026-11-10 22:09:38,788 INFO desktopenv.experiment] Result: 0.00"]) + "\n") + + patches = {"_vm": lambda *a: "vm", "_run": run, "_task": lambda *a: {"instruction": "", "proxy": False}, + "_assets": lambda *a: "assets"} + with mock.patch.multiple(self.driver, **patches), self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("a site's state (timed out)", str(raised.exception)) + self.assertIn("see the unit's nop.log", str(raised.exception)) + + def set_up(argv, cwd, env, log, *a, **kw): + with open(os.path.join(argv[-1], "result.txt"), "w") as f: + f.write("0.0\n") + with open(log, "w") as f: + f.write("[2026-11-10 22:00:01,000 INFO desktopenv.env] Environment is clean, skipping snapshot revert.\n" + "[2026-11-10 22:00:02,000 ERROR desktopenv.setup] Failed to launch application. Status code: x\n") + + with mock.patch.multiple(self.driver, **dict(patches, _run=set_up)): + self.assertEqual(self.driver.score("tasks__001", self.tmp)["setup_errors"], 1) + + +TRACE = ["[2026-11-10 20:00:00,000 INFO lib_run_single/1500-EnvProcess-1] [Instruction]: the task's words", + "[2026-11-10 20:00:01,000 ERROR run_multienv_m3/668-EnvProcess-1] Exception in EnvProcess-1 tasks/001: words", + "[2026-11-10 20:00:01,000 ERROR run_multienv_m3/669-EnvProcess-1] Traceback (most recent call last):", + ' File "/up/lib_run_single.py", line 1518, in run_single_example', + " env.reset(task_config=example)"] + + +class Failures(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.tmp = os.path.join(trial, "workspace") + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.logged = [] + + def run(argv, cwd, env, log, *a, **kw): + with open(log, "w") as f: + f.write("".join(ln + "\n" for ln in self.logged)) + where = os.path.join(cwd, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + os.makedirs(where, exist_ok=True) + with open(os.path.join(where, "traj.jsonl"), "w") as f: + f.write('{"Error": "tasks/001 - the task\'s words"}\n') + + self.real_run = self.driver._run + p = mock.patch.multiple(self.driver, _vm=lambda *a: "vm", _run=run, + _task=lambda *a: {"instruction": "", "proxy": False}, _assets=lambda *a: "assets") + p.start() + self.addCleanup(p.stop) + + def answer(self, *lines): + self.logged = list(lines) + with self.assertRaises(RuntimeError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertNotIn("words", str(raised.exception)) + return raised.exception + + def test_what_fails_before_the_upstream_starts_is_the_environments(self): + for e in (self.driver.NoAnswer("docker create got no answer in 60 s"), OSError("no space"), KeyError("proxy")): + with self.subTest(type(e).__name__), mock.patch.object(self.driver, "_vm", side_effect=e), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertTrue(str(raised.exception).startswith(type(e).__name__ + ": ")) + + def test_a_shared_lock_held_past_the_units_turn_limit_is_the_environments(self): + lock = os.path.join(self.tmp, "port-{}.lck") + fd = os.open(lock.format(9), os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + spare = self.driver.EVALUATE_SECONDS + self.driver.CLOSE_SECONDS + 1 + closing = threading.Timer(10, self.driver.CLOSED.set) + closing.start() + self.addCleanup(closing.cancel) + with mock.patch.dict(self.driver.GUEST_PORTS, {"001": (9,)}), mock.patch.object(self.driver, "PORT_LOCK", lock), \ + mock.patch.object(self.driver, "_turn_seconds", lambda: spare), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertIn("the unit's time ran out while another unit's VM held port 9", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + with mock.patch.dict(self.driver.GUEST_PORTS, {"001": (9,)}), mock.patch.object(self.driver, "PORT_LOCK", lock), \ + mock.patch.object(self.driver, "_score_seconds", lambda: self.driver.CLOSE_SECONDS + 1), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("the unit's time ran out while another unit's VM held port 9", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + given = [] + with mock.patch.object(self.driver, "_run", lambda *a, **kw: given.append(kw)), self.assertRaises(RuntimeError): + self.driver.score("tasks__001", self.tmp) + self.assertAlmostEqual(given[0]["until"] - time.time(), + self.driver._score_seconds() - 2 * self.driver.CLOSE_SECONDS, delta=5) + self.assertEqual(given[0]["grace"], self.driver.CLOSE_SECONDS) + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + self.addCleanup(p.kill) + t0 = time.time() + self.driver._started(p, os.path.join(self.tmp, "none.log"), "the null control's run", until=t0 + 1) + self.assertLess(time.time() - t0, 5) + self.assertIsNone(p.poll()) + + def test_a_traceback_the_runner_went_on_after_is_not_why_it_ended(self): + crash = [*TRACE, ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', "desktop_env.desktop_env.EnvironmentSetupError: x"] + phase, result = (STEP.replace(STEP[STEP.index("] ") + 2:], said) for said in ("Starting phase 2 of 2", "Result: 0.00")) + for then in (STEP, phase, result): + with self.subTest(then): + env = self.answer(*crash, then) + self.assertIs(type(env), RuntimeError) + self.assertIn("wrote no result.txt: no exception named", str(env)) + + def test_a_crash_on_the_vms_failed_last_screenshot_is_the_environments_and_no_other(self): + shot = "[2026-11-10 20:00:0{},000 {} python/{}-EnvProcess-1] {}" + got, lost = shot.format(2, "INFO", 467, "Got screenshot successfully"), shot.format(3, "ERROR", 480, "Failed to get screenshot.") + step = "[2026-11-10 20:00:01,000 INFO lib_run_single/1530-EnvProcess-1] Step 7: the task's words" + crash = [TRACE[2], ' File "/up/lib_run_single.py", line 1586, in run_single_example', " _f.write(obs['screenshot'])", + "TypeError: a bytes-like object is required, not 'NoneType'"] + env = self.answer(TRACE[0], step, got, lost, *crash) + self.assertIs(type(env), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: the VM stopped answering in step 7: its last screenshot failed", str(env)) + muse = [ln.replace("python/480", "desktopenv.pycontroller/480").replace("lib_run_single/", "desktopenv.experiment/") + for ln in (step, got, lost)] + self.assertIn("in step 7: its last screenshot failed", str(self.answer(TRACE[0], *muse, *crash))) + first = self.answer(TRACE[0], lost, *crash[:-1], "PIL.UnidentifiedImageError: cannot identify image file") + self.assertIn("the VM stopped answering before the first step", str(first)) + judged = self.answer(TRACE[0], step, lost, *crash[:2], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + crash[-1]) + self.assertIs(type(judged), self.driver.VerifierError) + reset = self.answer(TRACE[0], lost, *crash[:2], ' File "/up/desktop_env/desktop_env.py", line 300, in reset', "KeyError: 'x'") + self.assertIs(type(reset), self.driver.EnvironmentSetupError) + muse = self.answer(TRACE[0], step, lost, *crash[:2], ' File "/up/mm_agents/muse_spark_agent.py", line 605, in _screenshot_bytes', + "mm_agents.muse_spark_agent.StepError: words") + self.assertIn("in step 7: its last screenshot failed", str(muse)) + agent = ' File "/up/mm_agents/m3/agent.py", line 9, in predict' + for name, lines in (("another exception", [step, lost, *crash[:-1], "KeyError: 'screenshot'"]), + ("a TypeError of another kind", [step, lost, *crash[:-1], "TypeError: unsupported operand"]), + ("the agent's own None", [step, lost, *crash[:2], agent, crash[-1]])): + with self.subTest(name): + unknown = self.answer(TRACE[0], *lines) + self.assertIs(type(unknown), self.driver.CauseUnknownError) + self.assertIn("after the VM's last screenshot failed in step 7", str(unknown)) + for name, lines in (("a later screenshot came", [step, lost, got, *crash]), ("no screenshot failed", [step, got, *crash])): + with self.subTest(name): + self.assertIs(type(self.answer(TRACE[0], *lines)), RuntimeError) + + def test_a_runner_ended_by_a_signal_or_with_a_lost_worker_is_said_by_what_its_log_can_show(self): + log = os.path.join(self.tmp, "run.log") + signalled = [sys.executable, "-c", "import os, sys, signal; signal.signal(signal.SIGHUP, signal.SIG_DFL); print('Step 3: words\\nTraceback (most recent call last):\\nKeyError: 1', flush=True); os.kill(os.getpid(), int(sys.argv[1]))"] + aborted = [sys.executable, "-c", "import os; os.abort()"] + crashed = [sys.executable, "-c", "raise RuntimeError('words')"] + with mock.patch.object(self.driver, "_reaper", lambda *a, **kw: None): + for n in (1, 9, 15): + with contextlib.suppress(OSError): + os.remove(log) + with self.subTest(n), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.real_run(signalled + [str(n)], self.tmp, dict(os.environ), log, "the runner") + self.assertIn(f"the runner exited -{n}: ended by signal {n} (the whole log", str(raised.exception)) + for argv, kind, said in ((aborted, self.driver.CauseUnknownError, "exited -6: ended by signal 6, with no traceback"), + (crashed, RuntimeError, "exited 1: RuntimeError")): + os.remove(log) + with self.subTest(said), self.assertRaises(RuntimeError) as raised: + self.real_run(argv, self.tmp, dict(os.environ), log, "the runner") + self.assertIs(type(raised.exception), kind) + self.assertIn(said, str(raised.exception)) + at = "[2026-11-10 20:00:0{},000 INFO run_multienv_m3/{}] {}" + started, ended = at.format(0, "612-EnvProcess-1", "Process EnvProcess-1 started."), \ + at.format(5, "774-EnvProcess-1", "EnvProcess-1 cleaning up environment...") + finished = at.format(1, "879-MainProcess", "All tasks finished.") + lost = self.answer(started, finished, TRACE[0]) + self.assertIs(type(lost), self.driver.CauseUnknownError) + self.assertIn("wrote no result.txt: its worker process ended without its cleanup (the whole log", str(lost)) + self.assertIs(type(self.answer(started, *TRACE, " x", "RuntimeError: words")), self.driver.CauseUnknownError) + self.assertIs(type(self.answer(started, finished, TRACE[0], ended)), RuntimeError) + self.assertIs(type(self.answer(started, *TRACE, " x", "RuntimeError: words", ended)), RuntimeError) + self.assertIs(type(self.answer(finished, TRACE[0], ended)), RuntimeError) + term = at.format(6, "799-MainProcess", "Received signal 15. Gracefully shutting down...") + outside = self.answer(started, term, at.format(6, "799-EnvProcess-1", "Process 1 received signal 15. Shutting down...")) + self.assertIs(type(outside), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: ended by signal 15 (the whole log", str(outside)) + with open(os.path.join(self.tmp, "ended.log"), "w") as f: + f.write(f"{started}\n{term}\n") + self.assertIn("ended by signal 15 (", self.driver._tail(os.path.join(self.tmp, "ended.log"), 1)[1]) + own = at.format(6, "1051-MainProcess", "Unexpected error in main process: words") + self.assertIs(type(self.answer(started, own, *TRACE, " x", "RuntimeError: words", term, ended)), RuntimeError) + child = at.format(6, "799-EnvProcess-1", "Received signal 15. Gracefully shutting down...") + self.assertIs(type(self.answer(started, child, ended)), RuntimeError) + + def test_a_null_control_setup_that_fails_lets_the_units_lock_go(self): + with mock.patch.object(self.driver, "_task_file", side_effect=RuntimeError("no task")), self.assertRaises(RuntimeError): + self.driver.setup("tasks__001", self.tmp) + self.assertEqual(self.driver._OWN, []) + + def test_a_runner_with_no_result_is_said_by_its_last_tracebacks_known_class_only(self): + env = self.answer(*TRACE, ' File "/up/desktop_env/desktop_env.py", line 407, in reset', + "desktop_env.desktop_env.EnvironmentSetupError: Custom task setup failed: the task's words") + self.assertIs(type(env), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: EnvironmentSetupError (the whole log: ", str(env)) + started = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 186, in _start_emulator', + ' File "/up/desktop_env/providers/docker/provider.py", line 9, in start_emulator', + "requests.exceptions.ReadTimeout: the task's words") + self.assertIs(type(started), self.driver.EnvironmentSetupError) + self.assertIn("wrote no result.txt: ReadTimeout (the whole log: ", str(started)) + judged = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + "RuntimeError: Hosted state is unavailable: the task's words") + self.assertIs(type(judged), self.driver.VerifierError) + self.assertIn("RuntimeError (the whole log: ", str(judged)) + own = self.answer(*TRACE[:4], ' File "/up/mm_agents/m3.py", line 9, in reset', "PrivateWordsError: words") + self.assertIs(type(own), RuntimeError) + self.assertIn("an exception of the upstream's own (the whole log: ", str(own)) + self.assertIn("no exception named (the whole log: ", str(self.answer(*TRACE[:2]))) + quoted = self.answer(*TRACE[:4], ' File "/up/desktop_env/desktop_env.py", line 596, in evaluate', + "RuntimeError: words, in reset") + self.assertIs(type(quoted), self.driver.VerifierError) + + def test_a_runner_that_cannot_start_is_the_environments(self): + with mock.patch.object(self.driver.subprocess, "Popen", side_effect=BlockingIOError(11, "no process")), \ + self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.real_run([sys.executable, "-c", ""], self.tmp, {}, os.path.join(self.tmp, "x.log"), "the runner") + self.assertEqual(str(raised.exception), "the runner did not start: BlockingIOError: [Errno 11] no process") + + def test_the_null_controls_failure_after_its_setup_is_said_as_its_class(self): + def run(argv, cwd, env, log, *a, **kw): + with open(log, "w") as f: + f.write("".join(ln + "\n" for ln in TRACE[2:] + [ + ' File "/up/desktop_env/desktop_env.py", line 131, in __init__', + ' File "/up/desktop_env/desktop_env.py", line 186, in _start_emulator', + "docker.errors.APIError: the task's words"])) + + with mock.patch.object(self.driver, "_run", run), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.tmp) + self.assertIn("wrote no result.txt: an exception of the upstream's own", str(raised.exception)) + self.assertNotIn("words", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_gitlab.py b/app/tests/test_osworld2_gitlab.py new file mode 100644 index 0000000..6a6cfec --- /dev/null +++ b/app/tests/test_osworld2_gitlab.py @@ -0,0 +1,740 @@ +"""OSWorld2's GitLab, as gitlab_server.py and the driver reach it, before a fake GitLab on loopback. +With NULLIUS_OSWORLD2_GITLAB or NULLIUS_OSWORLD2_UPSTREAM set, also checked against them, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock +from urllib.parse import urlsplit + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +HOST = "gitlab.10.213.0.1.nip.io" +URL = f"http://{HOST}" +TOKEN = "t" * 24 +JUDGE = {"model": "anthropic/claude-sonnet-4.6", "base_url": "http://127.0.0.1:9/u1-judge/v1", "key": "k"} + + +def load(): + spec = importlib.util.spec_from_file_location("osworld2_gitlab", os.path.join(HERE, "gitlab_server.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "path", [HERE, *sys.path]): + spec.loader.exec_module(module) + return module + + +class Lab(BaseHTTPRequestHandler): + """A fake GitLab: an ``empty`` sign-in page, the token's ``admin``, its online ``runners``.""" + empty, admin, runners = False, True, [{"id": 1}] + + def log_message(self, *_): + pass + + def do_GET(self): + url = urlsplit(self.path) + if url.path == "/users/sign_in": + self.answer(200, b"" if self.empty else b"GitLab") + elif self.headers.get("PRIVATE-TOKEN") != TOKEN: + self.answer(401, b"{}") + elif url.path == "/api/v4/user": + self.answer(200, json.dumps({"is_admin": self.admin}).encode()) + elif url.path == "/api/v4/runners/all": + self.answer(200, json.dumps(self.runners).encode()) + else: + self.answer(404, b"{}") + + def answer(self, status, body): + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class Case(unittest.TestCase): + def setUp(self): + self.g = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + Lab.empty, Lab.admin, Lab.runners = False, True, [{"id": 1}] + server = ThreadingHTTPServer(("127.0.0.1", 0), Lab) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + for patch in (mock.patch.object(self.g, "PROXIES", {"http": f"http://127.0.0.1:{server.server_port}"}), + mock.patch.object(self.g.websites, "_resolves", lambda host: {"10.213.0.1"}), + mock.patch.object(self.g, "LOCK", os.path.join(self.tmp, "gitlab.lck")), + mock.patch.object(self.g.websites, "SITES_LOCK", os.path.join(self.tmp, "sites.lck"))): + patch.start() + self.addCleanup(patch.stop) + + +class Option(Case): + def test_none_gives_no_url_and_a_task_that_needs_gitlab_is_refused(self): + self.assertIsNone(self.g.url({})) + self.assertIsNone(self.g.needed("001", {})) + self.assertEqual(self.g.env("026", {}), {}) + with self.assertRaises(self.g.GitLabError) as raised: + self.g.needed("026", {self.g.OPTION: "none", self.g.TOKEN: TOKEN}) + self.assertIn("task 026 needs GitLab", str(raised.exception)) + self.assertIn("GitLab option", str(raised.exception)) + + def test_a_url_with_no_token_is_refused(self): + with self.assertRaises(self.g.GitLabError) as raised: + self.g.needed("041", {self.g.OPTION: HOST}) + self.assertIn("licence GITLAB_PRIVATE_TOKEN", str(raised.exception)) + + def test_the_url_is_gitlab_s_tasks_alone(self): + given = {self.g.OPTION: HOST, self.g.TOKEN: TOKEN} + self.assertEqual(self.g.needed("041", given), URL) + self.assertIsNone(self.g.needed("005", given)) + self.assertEqual(self.g.env("026", given), {"GITLAB_URL": URL}) + self.assertEqual(self.g.env("005", given), {}) + + def test_a_value_that_is_no_url_is_refused(self): + for bad in ("http://gitlab.x.io", "x.io/path", "X.IO", "-x.io", "x.io:0", "a" * 65): + with self.subTest(bad), self.assertRaises(self.g.GitLabError): + self.g.url({self.g.OPTION: bad}) + + def test_the_option_s_pattern_is_the_module_s(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + self.assertEqual(bench["options"]["gitlab"]["pattern"], "none|" + self.g.HOST.pattern) + from nullius.standard import definitions + self.assertIsNone(definitions.option_problem(bench["options"]["gitlab"], HOST)) + self.assertIsNotNone(definitions.option_problem(bench["options"]["gitlab"], URL)) + self.assertEqual([s for e in bench["environments"].values() for s in e["needs_secrets"]], + [{"name": self.g.TOKEN, "to": "driver", "summary": mock.ANY}]) + + +class Ready(Case): + def test_sign_in_token_and_runner(self): + self.assertEqual(self.g.ready(URL, TOKEN, runner=True), URL) + + def test_an_empty_sign_in_page_is_not_gitlab(self): + Lab.empty = True + with self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN) + self.assertIn("answered no page", str(raised.exception)) + + def test_another_token_or_no_administrator_is_refused(self): + for token, admin in (("x" * 24, True), (TOKEN, False)): + Lab.admin = admin + with self.subTest(admin=admin), self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, token) + self.assertIn("GITLAB_PRIVATE_TOKEN", str(raised.exception)) + self.assertNotIn(token, str(raised.exception)) + + def test_no_runner_online_fails_only_where_one_is_needed(self): + Lab.runners = [] + self.assertEqual(self.g.ready(URL, TOKEN), URL) + with self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN, runner=True) + self.assertIn("no runner online", str(raised.exception)) + + def test_no_answer_says_what_to_do(self): + with mock.patch.object(self.g, "PROXIES", {"http": "http://127.0.0.1:9"}), \ + self.assertRaises(self.g.GitLabError) as raised: + self.g.ready(URL, TOKEN) + self.assertIn("did not answer", str(raised.exception)) + + +class Held(Case): + def test_gitlab_s_tasks_run_one_at_a_time_and_the_lock_goes_to_the_reaper(self): + inside, released, order = threading.Event(), threading.Event(), [] + + def first(): + with self.g.held(URL, threading.Event()): + order.append("first") + inside.set() + released.wait(10) + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + closed = threading.Event() + closed.set() + with self.assertRaises(self.g.websites.WebsiteError) as raised: + with self.g.held(URL, closed): + pass + self.assertIn("another unit's GitLab task ran", str(raised.exception)) + with self.g.held(None, closed): + order.append("other") + released.set() + t.join(10) + with self.g.held(URL, threading.Event()): + self.assertEqual(len(self.g.websites.held_fds()), 2) + order.append("second") + self.assertEqual(order, ["first", "other", "second"]) + self.assertEqual(self.g.websites.held_fds(), ()) + + +class Shadow(unittest.TestCase): + def test_no_module_of_the_folder_hides_one_the_upstream_imports(self): + """A module here named as one the upstream imports would be loaded in its place by nop.py.""" + here = {n[:-3] for n in os.listdir(HERE) if n.endswith(".py")} + self.assertFalse(here & {"gitlab", "requests", "dotenv", "docker", "desktop_env", "mm_agents", "lib_run_single", + "task_loader"}) + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + if not up: + return + files = glob.glob(os.path.join(up, "*.py")) + _task_classes() + for d in ("desktop_env", "mm_agents"): + files += glob.glob(os.path.join(up, d, "**", "*.py"), recursive=True) + imported = set() + for p in files: + with open(p, encoding="utf-8", errors="replace") as f: + try: + tree = ast.parse(f.read()) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported |= {a.name.split(".")[0] for a in node.names} + elif isinstance(node, ast.ImportFrom) and not node.level and node.module: + imported.add(node.module.split(".")[0]) + self.assertGreater(len(files), 100) + self.assertFalse(here & imported) + + +class Share(unittest.TestCase): + def setUp(self): + self.g = load() + + def test_none_with_no_vm_limit_and_a_share_given(self): + self.assertIsNone(self.g.share({})) + self.assertEqual(self.g.share({self.g.SHARE_OPTION: "8:80", "NULLIUS_OPTION_WRITE_MBPS": "8"}), (8, 80)) + + def test_with_the_vms_limited_gitlab_needs_a_share(self): + for limited in ({"NULLIUS_OPTION_WRITE_MBPS": "8"}, {"NULLIUS_OPTION_WRITE_IOPS": "80"}): + with self.subTest(limited), self.assertRaises(self.g.GitLabError) as raised: + self.g.share(limited) + self.assertIn("GitLab write share", str(raised.exception)) + + def test_a_share_that_is_no_share_is_refused(self): + for bad in ("8", "8:", ":80", "0:80", "8:4", "8/80"): + with self.subTest(bad), self.assertRaises(self.g.GitLabError): + self.g.share({self.g.SHARE_OPTION: bad}) + + def test_the_option_s_pattern_is_the_module_s(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + self.assertEqual(bench["options"]["gitlab_write"]["pattern"], "none|" + self.g.SHARE.pattern) + + +class Allocation(unittest.TestCase): + def test_every_service_s_share_within_gitlab_s(self): + g = load() + out = g.allocation(8, 80) + self.assertEqual(set(out), {s for s, _ in g.SHARES}) + self.assertLessEqual(sum(v[0] for v in out.values()), 8 * g.websites.MB) + self.assertLessEqual(sum(v[1] for v in out.values()), 80) + self.assertEqual(out["GITLAB"], (6 * g.websites.MB, 60)) + for bad in ((0, 80), (8, 4)): + with self.subTest(bad), self.assertRaises(g.GitLabError): + g.allocation(*bad) + + +class Started(Exception): + pass + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_gitlab", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class Driver(unittest.TestCase): + """The driver's use of gitlab_server.py.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.g = self.driver.gitlab_server + self.envs, self.checked, self.held = [], [], [] + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + @contextlib.contextmanager + def held(value, closed): + self.held.append((value, closed)) + yield + + self.ups, self.order = [], [] + + @contextlib.contextmanager + def warden(dir): + self.order.append(("warden", dir)) + try: + yield + finally: + self.order.append(("let go", dir)) + + for target, name, fake in ((self.driver, "_vm", lambda dir, domain, eid: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_task_file", lambda *a: None), + (self.g, "held", held), (self.g, "prepared", lambda: self.checked.append(1) or "/s"), + (self.g, "unit", lambda where, dir: os.path.join(dir, "gitlab.limits")), + (self.g, "up", lambda *a: self.ups.append(a) or self.order.append(("up",))), + (self.driver, "_warden", warden)): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + p = mock.patch.dict(os.environ, {self.g.OPTION: HOST, self.g.TOKEN: TOKEN, + self.driver.websites.OPTION: "10.213.0.1.nip.io"}) + p.start() + self.addCleanup(p.stop) + p = mock.patch.object(self.driver.websites, "held", lambda *a: contextlib.nullcontext()) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + + def test_a_task_that_needs_gitlab_is_refused_without_it_before_anything_runs(self): + with mock.patch.dict(os.environ, {self.g.OPTION: "none"}): + setup = self.driver.EnvironmentSetupError + for start, kind in ((lambda: self.driver.setup("tasks__026", self.tmp), self.g.GitLabError), + (lambda: self.answer("026"), setup), + (lambda: self.driver.score("tasks__041", self.tmp, judge=JUDGE), setup)): + with self.assertRaises(kind) as raised: + start() + self.assertEqual(kind is setup, str(raised.exception).startswith("GitLabError: ")) + self.assertEqual(self.envs, []) + + def test_the_url_and_token_reach_gitlab_s_tasks_alone_and_gitlab_is_held(self): + for start in (lambda: self.answer("026"), lambda: self.driver.score("tasks__041", self.tmp, judge=JUDGE)): + with self.assertRaises(Started): + start() + self.assertEqual((self.envs[-1]["GITLAB_URL"], self.envs[-1][self.g.TOKEN]), (URL, TOKEN)) + with self.assertRaises(Started): + self.answer("001") + self.assertNotIn("GITLAB_URL", self.envs[-1]) + self.assertNotIn(self.g.TOKEN, self.envs[-1]) + self.assertEqual(self.held, [(URL, mock.ANY)] * 2 + [(None, mock.ANY)]) + limits = os.path.join(self.tmp, "gitlab.limits") + self.assertEqual(self.ups, [("/s", limits, mock.ANY)] * 2) + self.assertIsNotNone(self.held[0][-1].until) + self.assertIsNotNone(self.ups[0][-1].until) + self.assertEqual(self.order, [("warden", self.tmp), ("up",), ("let go", self.tmp)] * 2) + + def test_gitlab_is_checked_among_what_a_vm_start_needs(self): + for name in ("vm_image", "_provider_image", "_write_limit"): + p = mock.patch.object(self.driver, name, lambda: "image") + p.start() + self.addCleanup(p.stop) + with mock.patch.object(self.driver, "_nat", lambda: True), \ + mock.patch.object(self.driver.os, "listdir", lambda p: ["x"]), \ + mock.patch.object(self.driver.os.path, "isdir", lambda p: True), \ + mock.patch.object(self.driver.websites, "ready", lambda *a, **kw: None), \ + mock.patch.object(self.driver._startup(), "network", lambda docker: "nullius-osworld2"): + self.driver._ready("tasks", "001") + self.driver._ready("tasks", "026") + self.driver._ready("tasks", "041") + self.assertEqual(self.checked, [1, 1]) + + def test_a_hosts_snapshot_carries_the_module(self): + from pathlib import Path + + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + for name in ("gitlab_server.py", "gitlab.compose.yml"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + self.assertEqual(json.load(f)["driver"]["command"][-1], "{here}/gitlab.compose.yml") + + +def _stack(): + stack = os.environ.get("NULLIUS_OSWORLD2_GITLAB", "") + return stack if stack and shutil.which("docker") and os.path.isfile(os.path.join(stack, "docker-compose.yml")) else "" + + +@unittest.skipUnless(_stack(), "NULLIUS_OSWORLD2_GITLAB names no Task-Web/gitlab, or no docker compose") +class Override(unittest.TestCase): + def config(self, *args): + g = load() + shares = g.allocation(8, 80) + settings = {"GITLAB_URL": URL, "GITLAB_PRIVATE_TOKEN": g.UNUSED, "OSWORLD2_GITLAB_HOST": URL[7:], + "OSWORLD2_GITLAB_INGRESS": "osworld2-web_web", "OSWORLD2_GITLAB_DISK": "/dev/sda", + "OSWORLD2_GITLAB_RB": "/x/gitlab.rb", + **{f"OSWORLD2_GITLAB_{s}_{k}": str(v[i]) for s, v in shares.items() for i, k in enumerate(("BPS", "IOPS"))}} + said = subprocess.run(["docker", "compose", "-p", g.PROJECT, "-f", os.path.join(_stack(), "docker-compose.yml"), + "-f", g.OVERRIDE, *args, "config", "--format", "json"], capture_output=True, text=True, + env=dict(os.environ, **settings), timeout=120) + self.assertEqual(said.returncode, 0, said.stderr) + return g, shares, json.loads(said.stdout)["services"] + + def test_the_project_as_compose_reads_it(self): + g, shares, services = self.config() + self.assertEqual(sorted(services), ["dind", "gitlab", "gitlab-runner"]) + for name, service in services.items(): + share = shares[{"gitlab": "GITLAB", "gitlab-runner": "RUNNER", "dind": "DIND"}[name]] + self.assertEqual(service.get("blkio_config"), { + "device_write_bps": [{"Path": "/dev/sda", "Rate": str(share[0])}], + "device_write_iops": [{"Path": "/dev/sda", "Rate": str(share[1])}]}, name) + self.assertFalse(service.get("ports"), name) + self.assertNotIn("container_name", service, name) + self.assertEqual(service.get("restart"), "no", name) + self.assertRegex(service["image"], r"@sha256:[0-9a-f]{64}$", name) + self.assertFalse([v for v in service.get("volumes") or [] if "docker.sock" in str(v.get("source"))], name) + self.assertEqual(services["gitlab"]["labels"]["caddy"], f"{URL}, http://*.{URL[7:]}") + self.assertEqual(list(services["gitlab"]["networks"]), ["web"]) + self.assertEqual(list(services["gitlab-runner"]["networks"]), ["ci"]) + self.assertTrue(services["dind"]["privileged"]) + + def test_the_upstream_s_token_service_is_disarmed_and_has_no_socket(self): + g, _, services = self.config("--profile", "upstream-init") + self.assertEqual(sorted(services), ["dind", "gitlab", "gitlab-init-token", "gitlab-runner"]) + for name, service in services.items(): + self.assertFalse([v for v in service.get("volumes") or [] if "docker.sock" in str(v.get("source"))], name) + self.assertEqual(services["gitlab-init-token"]["entrypoint"], ["/bin/false"]) + self.assertFalse(services["gitlab-init-token"].get("command")) + + def test_the_upstream_s_token_step_is_the_one_gitlab_server_py_runs(self): + g = load() + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + stack = os.path.join(tmp, "gitlab") + os.symlink(_stack(), stack) + with open(stack + g.ENV_FILE, "w") as f: + f.write(f"GITLAB_URL={URL}\nGITLAB_PRIVATE_TOKEN={g.UNUSED}\n") + script = g._upstream_script(stack) + self.assertEqual(script.count(f'-e {g.TOKEN}="$${g.TOKEN}" gitlab '), 1) + self.assertEqual(script.count("docker exec gitlab "), 1) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + def test_the_tasks_that_need_gitlab(self): + g = load() + found = set() + for p in _task_classes(): + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + if any(isinstance(n, ast.ImportFrom) and (n.module or "").endswith("controllers.gitlab") + for n in ast.walk(tree)): + found.add(os.path.basename(p)[5:-3]) + self.assertEqual(sorted(found), sorted(g.TASKS)) + self.assertLess(g.RUNNER_TASKS, g.TASKS) + + +class Fake: + """A finished process as subprocess.run returns it.""" + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode, self.stdout, self.stderr = returncode, stdout, stderr + + +class Lifecycle(unittest.TestCase): + """configure, the token step, the runner, the volumes' copy, mark and reap, and the lock, on fakes.""" + + def setUp(self): + self.g = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.stack = os.path.join(self.tmp, "gitlab") + self.calls = [] + p = mock.patch.object(self.g.websites, "SITES_LOCK", os.path.join(self.tmp, "sites.lck")) + p.start() + self.addCleanup(p.stop) + for name, fake in (("LOCK", os.path.join(self.tmp, "gitlab.lck")), + ("_docker", lambda *a, **kw: self.calls.append((a, kw)) or Fake()), + ("_pinned", lambda stack: os.path.abspath(stack)), + ("_container", lambda stack, service: f"id-{service}")): + p = mock.patch.object(self.g, name, fake) + p.start() + self.addCleanup(p.stop) + p = mock.patch.dict(os.environ, {self.g.TOKEN: TOKEN}) + p.start() + self.addCleanup(p.stop) + + def configure(self): + with mock.patch.object(self.g.websites, "_network", lambda: "nullius-osworld2"), \ + mock.patch.object(self.g.websites, "_root_disk", lambda: ("/var/lib/docker", "/dev/sda")): + return self.g.configure(self.stack, "10.213.0.1.nip.io", 8, 80) + + def test_configure_writes_no_token_and_its_placeholder_is_no_token(self): + path, settings = self.configure() + with open(path, encoding="utf-8") as f: + text = f.read() + self.assertNotIn(TOKEN, text) + self.assertIn(f"GITLAB_PRIVATE_TOKEN={self.g.UNUSED}\n", text) + self.assertEqual(settings["GITLAB_URL"], URL) + self.assertEqual(settings["OSWORLD2_GITLAB_SHARE"], "stack") + with mock.patch.dict(os.environ, {self.g.TOKEN: self.g.UNUSED}), self.assertRaises(self.g.GitLabError): + self.g._token() + + def test_the_token_step_passes_the_token_by_name(self): + script = ('until docker exec gitlab gitlab-rails runner "x"; do sleep 1; done\n' + f'docker exec -e {self.g.TOKEN}="$${self.g.TOKEN}" gitlab gitlab-rails runner "y"\n') + ran = [] + with mock.patch.object(self.g, "_upstream_script", lambda stack: script), \ + mock.patch.object(self.g.subprocess, "run", lambda argv, **kw: ran.append((argv, kw)) or Fake()): + self.g._make_token(self.stack) + (argv, kw), = ran + self.assertNotIn(TOKEN, " ".join(argv)) + self.assertEqual(kw["env"][self.g.TOKEN], TOKEN) + self.assertIn(f"docker exec -e {self.g.TOKEN} id-gitlab gitlab-rails", argv[-1]) + self.assertIn("until docker exec id-gitlab gitlab-rails", argv[-1]) + self.assertNotIn("$$", argv[-1]) + with mock.patch.object(self.g, "_upstream_script", lambda stack: "echo other"), \ + self.assertRaises(self.g.GitLabError): + self.g._make_token(self.stack) + + def test_the_runner_runs_jobs_unprivileged_in_the_project_s_docker(self): + asked = [] + + def request(path, value, token=None, data=None, method=None): + asked.append((path, method)) + return 201, json.dumps({"id": 7, "token": "glrt-secret"}).encode() + + with mock.patch.object(self.g, "_request", request): + self.assertEqual(self.g._register(self.stack, URL), 7) + (argv, kw), = self.calls + self.assertNotIn("--docker-privileged", argv) + self.assertNotIn("glrt-secret", " ".join(argv)) + self.assertEqual(kw["env"]["CI_SERVER_TOKEN"], "glrt-secret") + for flag in ("--docker-tlsverify", self.g.DOCKER_HOST, "/certs/client"): + self.assertIn(flag, argv) + with mock.patch.object(self.g, "_docker", lambda *a, **kw: Fake(1, stderr="FATAL: no")), \ + self.assertRaises(self.g.GitLabError): + self.g._register(self.stack, URL) + self.assertEqual(asked[-1], ("/api/v4/runners/7", "DELETE")) + + def test_the_copy_runs_under_gitlab_s_limit_with_its_labels(self): + settings = {"OSWORLD2_GITLAB_DISK": "/dev/sda", "OSWORLD2_GITLAB_GITLAB_BPS": "6291456", + "OSWORLD2_GITLAB_GITLAB_IOPS": "60", "OSWORLD2_GITLAB_SHARE": "unit"} + with mock.patch.object(self.g, "compose", lambda *a, **kw: Fake(stdout="gitlab/gitlab-ce@sha256:x\n")): + self.g._copy(self.stack, settings, "tar -x", keep=False) + self.g._copy(self.stack, settings, "tar -c", keep=True) + restore, keep = [a for a, _ in self.calls] + for flag in ("--network none", "--label com.docker.compose.project=osworld2-gitlab", + "--label nullius.osworld2.share=unit", "--device-write-bps /dev/sda:6291456", + "--device-write-iops /dev/sda:60"): + self.assertIn(flag, " ".join(restore)) + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot:ro", restore) + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot", keep) + + def test_a_unit_s_mark_and_limits_and_its_reap(self): + self.configure() + environ = {self.g.SHARE_OPTION: "8:80", "NULLIUS_OPTION_WRITE_MBPS": "8"} + limits = self.g.unit(self.stack, self.tmp, environ) + settings = self.g._settings(self.stack, limits) + self.assertEqual(settings["OSWORLD2_GITLAB_SHARE"], "unit") + self.assertEqual(settings["OSWORLD2_GITLAB_GITLAB_IOPS"], "60") + gone = [] + with mock.patch.object(self.g, "down", lambda stack, limits=None: gone.append((stack, limits))): + self.assertTrue(self.g.reap(self.tmp)) + self.assertFalse(self.g.reap(self.tmp)) + self.assertEqual(gone, [(self.stack, limits)]) + self.assertIsNone(self.g.unit(self.stack, self.tmp, {})) + + def test_up_down_and_init_by_hand_refuse_while_a_unit_holds_gitlab(self): + ran = [] + with mock.patch.object(self.g, "up", lambda *a, **kw: ran.append("up")), \ + mock.patch.object(self.g, "down", lambda *a, **kw: ran.append("down")), \ + mock.patch.object(self.g, "init", lambda *a, **kw: ran.append("init")): + with self.g.held(URL, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()) as err: + for op in ("up", "down", "init"): + self.assertEqual(self.g.main([op, self.stack]), 1) + self.assertIn("a unit's GitLab task runs", err.getvalue()) + self.assertEqual(ran, []) + with mock.patch("sys.stdout", new=io.StringIO()): + self.assertEqual(self.g.main(["down", self.stack]), 0) + self.assertEqual(ran, ["down"]) + + def test_compose_by_hand_changes_nothing_while_a_unit_holds_gitlab(self): + ran = [] + with mock.patch.object(self.g, "compose", lambda stack, args, **kw: ran.append(args) or 0): + with self.g.held(URL, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()): + for args in (["down", "--volumes"], ["--profile", "upstream-init", "up", "-d"], ["-p", "ps", "rm"], []): + self.assertEqual(self.g.main(["compose", self.stack, *args]), 1, args) + for args in (["ps"], ["--profile", "upstream-init", "config"], ["--", "logs", "gitlab"]): + self.assertEqual(self.g.main(["compose", self.stack, *args]), 0, args) + self.assertEqual(self.g.main(["compose", self.stack, "down"]), 0) + self.assertEqual(ran, [["ps"], ["--profile", "upstream-init", "config"], ["logs", "gitlab"], ["down"]]) + + def test_one_gitlab_task_at_a_time_whoever_runs_it(self): + locks = [] + for uid in (1000, 1001): + with mock.patch.object(os, "getuid", lambda: uid): + locks.append(load().LOCK) + self.assertEqual(locks[0], locks[1]) + given = [] + with mock.patch.object(self.g, "_pinned", lambda stack: stack), mock.patch.object(self.g, "_settings", lambda *a: {}), \ + mock.patch.object(self.g.websites, "_compose_version", lambda: None), \ + mock.patch.object(self.g.subprocess, "run", lambda *a, **kw: given.append(kw["pass_fds"]) or mock.Mock()): + with self.g._alone(): + fds = self.g.websites.held_fds() + self.g.compose(self.stack, ["down"], capture=False) + self.assertEqual(len(fds), 2) + self.assertEqual(given, [fds]) + + def test_the_run_s_gitlab_host_must_be_the_stack_s(self): + self.configure() + with mock.patch.object(self.g, "stack", lambda: self.stack): + self.assertEqual(self.g.prepared({self.g.OPTION: HOST}), self.stack) + with self.assertRaises(self.g.GitLabError) as raised: + self.g.prepared({self.g.OPTION: "gitlab.example.org"}) + self.assertIn("set the GitLab option to the stack's host", str(raised.exception)) + + def test_the_stack_s_folder_is_read_from_the_snapshot_read_only(self): + with mock.patch.object(self.g, "_docker", lambda *a, **kw: self.calls.append(a) or Fake(stdout="/srv/gitlab\n")): + self.assertEqual(self.g.stack(), "/srv/gitlab") + self.assertIn(f"{self.g.SNAPSHOT}:/snapshot:ro", self.calls[-1]) + self.assertIn("none", self.calls[-1]) + for said in (Fake(1), Fake(stdout="relative\n")): + with mock.patch.object(self.g, "_docker", lambda *a, **kw: said), self.assertRaises(self.g.GitLabError): + self.g.stack() + + def test_no_override_is_one_line_saying_what_to_do(self): + with mock.patch.object(self.g, "OVERRIDE", os.path.join(self.tmp, "none.yml")), \ + self.assertRaises(self.g.GitLabError) as raised: + self.g._argv(self.stack, ["ps"]) + self.assertIn("install Nullius again", str(raised.exception)) + + +GIT = """#!/bin/sh +case "$*" in *rev-parse*) echo 8655d651722f4254e59e813de9f68a6732ea525c ;; esac +exit 0 +""" +FAKE_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$*" in + *"version --short"*) echo 2.40.3 ;; + *" down "*) sleep 2; echo "$*" >> "$FAKE_DOCKER_LOG.done" ;; +esac +exit 0 +""" +KILLED = """ +import importlib.util, os, sys, time +here, lock, sites = sys.argv[1:4] +spec = importlib.util.spec_from_file_location("osworld2_driver_killed", os.path.join(here, "driver.py")) +driver = importlib.util.module_from_spec(spec) +sys.argv = ["driver.py"] +sys.path.insert(0, here) +spec.loader.exec_module(driver) +driver.gitlab_server.LOCK, driver.websites.SITES_LOCK = lock, sites +driver.gitlab_server.prepared = lambda: os.environ["STACK"] +driver.gitlab_server.up = lambda *a: (print("up", flush=True), time.sleep(600)) +with driver._gitlab("041", os.environ["UNIT"]): + pass +""" + + +class Warden(unittest.TestCase): + """The driver killed while GitLab starts: its warden removes GitLab, then lets the locks go.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_ = os.path.join(self.tmp, "bin") + os.makedirs(bin_) + for name, text in (("git", GIT), ("docker", FAKE_DOCKER)): + with open(os.path.join(bin_, name), "w") as f: + f.write(text) + os.chmod(os.path.join(bin_, name), 0o755) + self.stack, self.unit = os.path.join(self.tmp, "gitlab"), os.path.join(self.tmp, "unit") + os.makedirs(self.stack) + os.makedirs(self.unit) + with open(self.stack + ".osworld2.env", "w") as f: + f.write(f"GITLAB_URL={URL}\n") + self.log = os.path.join(self.tmp, "docker.log") + self.lock, self.sites = os.path.join(self.tmp, "gitlab.lck"), os.path.join(self.tmp, "sites.lck") + self.env = dict(os.environ, PATH=bin_ + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.log, + NULLIUS_UPSTREAM=self.tmp, STACK=self.stack, UNIT=self.unit, + NULLIUS_OPTION_GITLAB=HOST, GITLAB_PRIVATE_TOKEN=TOKEN) + + def free(self, path, mode): + import fcntl + fd = os.open(path, os.O_RDWR | os.O_CREAT) + try: + fcntl.flock(fd, mode | fcntl.LOCK_NB) + return True + except BlockingIOError: + return False + finally: + os.close(fd) + + def test_the_driver_killed_during_up_leaves_gitlab_to_its_warden(self): + import fcntl + import signal + p = subprocess.Popen([sys.executable, "-c", KILLED, HERE, self.lock, self.sites], env=self.env, + stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: p.poll() is None and p.kill()) + self.assertEqual(p.stdout.readline().strip(), "up") + self.assertFalse(self.free(self.lock, fcntl.LOCK_EX)) + self.assertFalse(self.free(self.sites, fcntl.LOCK_EX)) + p.send_signal(signal.SIGKILL) + p.wait(10) + seen_held = False + t0 = time.time() + while not os.path.exists(self.log + ".done") and time.time() - t0 < 60: + seen_held = seen_held or not self.free(self.lock, fcntl.LOCK_EX) + time.sleep(0.2) + self.assertTrue(os.path.exists(self.log + ".done"), "the warden removed no GitLab") + self.assertTrue(seen_held, "the lock was let go before GitLab was removed") + while not self.free(self.lock, fcntl.LOCK_EX) and time.time() - t0 < 90: + time.sleep(0.5) + self.assertTrue(self.free(self.lock, fcntl.LOCK_EX)) + self.assertTrue(self.free(self.sites, fcntl.LOCK_EX)) + with open(self.log) as f: + said = f.read() + self.assertIn("compose -p osworld2-gitlab", said) + self.assertIn("down --volumes --remove-orphans", said) + self.assertFalse(os.path.exists(os.path.join(self.unit, "gitlab.stack"))) + + +class Sites(Case): + """GitLab's way in is the mock websites' Caddy: a GitLab unit and a hand operation keep a reset away.""" + + def test_a_reset_or_smoke_is_refused_while_a_gitlab_unit_runs(self): + with self.g.held(URL, threading.Event()): + with self.assertRaises(self.g.websites.WebsiteError): + with self.g.websites._alone(): + pass + self.assertEqual(len(self.g.websites.held_fds()), 2) + with self.g.websites._alone(): + pass + + def test_gitlab_by_hand_and_a_unit_wait_while_the_sites_are_reset(self): + closed = threading.Event() + closed.set() + with self.g.websites._alone(): + with self.assertRaises(self.g.websites.WebsiteError) as raised: + with self.g._alone(): + pass + self.assertIn("mock websites are being reset", str(raised.exception)) + with self.assertRaises(self.g.websites.WebsiteError): + with self.g.held(URL, closed): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_guest_disk.py b/app/tests/test_osworld2_guest_disk.py new file mode 100644 index 0000000..ef82dbd --- /dev/null +++ b/app/tests/test_osworld2_guest_disk.py @@ -0,0 +1,249 @@ +"""OSWorld2's guest disk on the docker provider: at least GUEST_GB, in nop.py and the Original's runner alike. +A fake desktop_env stands in.""" +from __future__ import annotations + +import importlib.machinery +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +DESKTOP_ENV = """import json, os +MAX_RETRIES = 5 +class EnvironmentSetupError(RuntimeError): + pass +class DesktopEnv: + def __init__(self, provider_name="vmware", region=None, path_to_vm=None, snapshot_name="init_state", + action_space="pyautogui", cache_dir="cache", screen_size=(1920, 1080), headless=False, + require_a11y_tree=True, require_terminal=False, os_type="Ubuntu", enable_proxy=False, + client_password="", instance_type=None, volume_size=None, use_public_ip=False, + force_disable_vnc=False, force_disable_recording=False): + with open(os.environ["FAKE_SIZES"], "a") as f: + f.write(json.dumps([provider_name, volume_size]) + "\\n") + def close(self): + pass +""" +TASK_LOADER = """import json, os +def find_task_class_path(*a): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, **kw): + return dict(json.loads(os.environ["FAKE_TASK"]), instruction="", proxy=False) +""" +LIB_RUN_SINGLE = """def run_single_example(*a, **kw): + pass +""" +RUNNER = """import json, multiprocessing, os, sys +from desktop_env.desktop_env import DesktopEnv +def worker(size): + DesktopEnv(path_to_vm="vm", action_space="pyautogui", provider_name=sys.argv[1], region=None, + snapshot_name="init_state", screen_size=(1920, 1080), headless=True, os_type="Ubuntu", + require_a11y_tree=False, enable_proxy=True, client_password="pw", instance_type=None, + volume_size=size, use_public_ip=False, force_disable_vnc=True, force_disable_recording=True) +if __name__ == "__main__": + size = json.loads(os.environ["FAKE_TASK"]).get("volume_size") + for how in ("fork", "spawn"): + p = multiprocessing.get_context(how).Process(target=worker, args=(size,)) + p.start() + p.join() + assert p.exitcode == 0, how +""" + + +DOCKER_PY = """class ContainerCollection: + def create(self, image, **kwargs): + pass +""" +AGAIN = """import importlib, json, os, runpy, sys +import desktop_env.desktop_env as module +runpy.run_path(os.environ["SITE"], run_name="sitecustomize") +runpy.run_path(os.environ["SITE"], run_name="sitecustomize") +module.DesktopEnv(provider_name="docker") +module = importlib.reload(module) +module.DesktopEnv(provider_name="docker") +from docker.models.containers import ContainerCollection +layers, create = 0, ContainerCollection.create +while hasattr(create, "__wrapped__"): + layers, create = layers + 1, create.__wrapped__ +print(json.dumps({"finders": sum(bool(getattr(f, "_nullius_osworld2", False)) for f in sys.meta_path), + "layers": layers})) +""" + + +def fake_class(): + class DesktopEnv: + def __init__(self, provider_name="docker", volume_size=None): + self.volume_size = volume_size + + return DesktopEnv + + +def load_site(): + spec = importlib.util.spec_from_file_location("osworld2_startup_disk", os.path.join(STARTUP, "sitecustomize.py")) + site = importlib.util.module_from_spec(spec) + spec.loader.exec_module(site) + return site + + +class Size(unittest.TestCase): + def setUp(self): + self.site = load_site() + + def test_the_floor_is_the_aws_providers_root(self): + self.assertEqual(self.site.GUEST_GB, 40) + + def test_docker_gets_the_floor_or_the_tasks_larger_size(self): + for given, wanted in ((None, 40), (0, 40), (30, 40), (40, 40), (60, 60), ("60", 60)): + with self.subTest(given=given): + self.assertEqual(self.site.guest_gb("docker", given), wanted) + self.assertEqual(self.site.guest_gb(" Docker ", None), 40) + + def test_another_provider_is_left_alone(self): + for provider in ("aws", "vmware", "virtualbox"): + for given in (None, 30, 60): + with self.subTest(provider=provider, given=given): + self.assertEqual(self.site.guest_gb(provider, given), given) + + def test_a_module_already_loaded_is_wrapped_and_one_finder_added(self): + module = types.SimpleNamespace(DesktopEnv=fake_class()) + with mock.patch.dict(sys.modules, {self.site.DESKTOP_ENV: module}), mock.patch.object(sys, "meta_path", []): + self.site.wrap_desktop_env() + load_site().wrap_desktop_env() + self.assertEqual(len(sys.meta_path), 1) + self.assertEqual(module.DesktopEnv().volume_size, 40) + + def test_the_finders_loader_is_wrapped_not_changed(self): + """A loader whose methods cannot be set, or that loads other modules too, is left as it was.""" + class Slotted: + __slots__ = () + + def create_module(self, spec): + return None + + def exec_module(self, module): + module.DesktopEnv = fake_class() + + loader = Slotted() + finder = types.SimpleNamespace(find_spec=lambda name, path, target=None: importlib.machinery.ModuleSpec( + name, loader)) + with mock.patch.object(sys, "meta_path", [self.site._Floor(), finder]): + spec = sys.meta_path[0].find_spec(self.site.DESKTOP_ENV, None) + self.assertIsNot(spec.loader, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertEqual(module.DesktopEnv().volume_size, 40) + other = types.ModuleType("other") + loader.exec_module(other) + self.assertIsNone(other.DesktopEnv().volume_size) + + def test_a_class_without_those_arguments_is_left_alone(self): + class DesktopEnv: + def __init__(self, provider="docker", size=None): + pass + + init = DesktopEnv.__init__ + self.site.floor_desktop_env(types.SimpleNamespace(DesktopEnv=DesktopEnv)) + self.assertIs(DesktopEnv.__init__, init) + + def test_the_class_is_wrapped_once_however_it_is_called(self): + made = [] + + class DesktopEnv: + def __init__(self, provider_name="vmware", region=None, path_to_vm=None, volume_size=None): + made.append((provider_name, volume_size, path_to_vm)) + + module = types.SimpleNamespace(DesktopEnv=DesktopEnv) + self.site.floor_desktop_env(module) + wrapped = DesktopEnv.__init__ + self.site.floor_desktop_env(module) + self.assertIs(DesktopEnv.__init__, wrapped) + DesktopEnv("docker", None, "vm") + DesktopEnv("docker", None, "vm", 60) + DesktopEnv(provider_name="docker", volume_size=30) + DesktopEnv() + DesktopEnv("aws", volume_size=None) + self.assertEqual(made, [("docker", 40, "vm"), ("docker", 60, "vm"), ("docker", 40, None), + ("vmware", None, None), ("aws", None, None)]) + + +class BothPaths(unittest.TestCase): + """nop.py and a runner built as run_multienv_m3.py, each with the startup folder first on PYTHONPATH.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.up = os.path.join(self.tmp, "up") + os.makedirs(os.path.join(self.up, "desktop_env")) + for name, text in (("task_loader.py", TASK_LOADER), ("lib_run_single.py", LIB_RUN_SINGLE), + ("runner.py", RUNNER), (os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV)): + with open(os.path.join(self.up, name), "w") as f: + f.write(text) + self.out = os.path.join(self.tmp, "out") + os.makedirs(self.out) + + def sizes(self, argv, task, **env): + log = os.path.join(self.tmp, "sizes") + if os.path.exists(log): + os.remove(log) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + environ.update(PYTHONPATH=os.pathsep.join([STARTUP, self.up]), FAKE_SIZES=log, FAKE_TASK=json.dumps(task), + **env) + done = subprocess.run([sys.executable, *argv], cwd=self.up, env=environ, capture_output=True, text=True, + timeout=60) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertNotIn("Error in sitecustomize", done.stderr) + with open(log) as f: + return [json.loads(ln) for ln in f.read().splitlines()] + + def nop(self, task, **env): + return self.sizes([os.path.join(HERE, "nop.py"), "vm", "pw", "examples", "tasks", "001", self.out], task, + **env) + + def test_the_null_control_and_the_original_setup_get_the_same_disk(self): + for task, wanted in (({}, 40), ({"volume_size": 30}, 40), ({"volume_size": 60}, 60)): + with self.subTest(task=task): + self.assertEqual(self.nop(task), [["docker", wanted]]) + self.assertEqual(self.sizes(["runner.py", "docker"], task), [["docker", wanted]] * 2) + + def test_another_provider_is_left_alone_on_the_runners_path(self): + self.assertEqual(self.sizes(["runner.py", "aws"], {"volume_size": 30}), [["aws", 30]] * 2) + + def test_another_vm_provider_option_wraps_nothing(self): + self.assertEqual(self.nop({}, NULLIUS_OPTION_VM_PROVIDER="aws"), [["docker", None]]) + + +class Again(unittest.TestCase): + """desktop_env imported before the startup module runs, the startup module run twice, then a reload.""" + def test_a_reload_keeps_the_floor_and_a_second_run_wraps_nothing_twice(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + for name, text in ((os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV), + (os.path.join("docker", "__init__.py"), ""), (os.path.join("docker", "models", "__init__.py"), ""), + (os.path.join("docker", "models", "containers.py"), DOCKER_PY), ("again.py", AGAIN)): + os.makedirs(os.path.dirname(os.path.join(tmp, name)), exist_ok=True) + with open(os.path.join(tmp, name), "w") as f: + f.write(text) + environ = {k: v for k, v in os.environ.items() if not k.startswith("NULLIUS_OPTION_")} + environ.update(PYTHONPATH=tmp, FAKE_SIZES=os.path.join(tmp, "sizes"), + SITE=os.path.join(STARTUP, "sitecustomize.py")) + done = subprocess.run([sys.executable, "again.py"], cwd=tmp, env=environ, capture_output=True, text=True, + timeout=60) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertEqual(json.loads(done.stdout), {"finders": 1, "layers": 1}) + with open(os.path.join(tmp, "sizes")) as f: + self.assertEqual([json.loads(ln) for ln in f.read().splitlines()], [["docker", 40]] * 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_guest_ports.py b/app/tests/test_osworld2_guest_ports.py new file mode 100644 index 0000000..672c20c --- /dev/null +++ b/app/tests/test_osworld2_guest_ports.py @@ -0,0 +1,154 @@ +"""OSWorld2's guest ports on 127.0.0.1, one VM at a time per machine, locked until the VM is gone.""" +from __future__ import annotations + +import fcntl +import importlib.util +import os +import shutil +import socket +import sys +import tempfile +import threading +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") + + +def load(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_ports", os.path.join(HERE, "driver.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(module) + return module + + +def unused(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class GuestPorts(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load(self.tmp) + self.port = unused() + self.driver.PORT_LOCK = os.path.join(self.tmp, "port-{}.lck") + self.driver.GUEST_PORTS = {"082": (self.port,)} + self.driver.CLOSED = threading.Event() + self.lock = os.path.join(self.tmp, f"port-{self.port}.lck") + + def held(self): + fd = os.open(self.lock, os.O_RDONLY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return False + except BlockingIOError: + return True + finally: + os.close(fd) + + def test_another_task_takes_no_lock_and_names_no_port(self): + with self.driver._published("001") as env: + self.assertEqual(env, {}) + self.assertEqual(self.driver._PORTS, []) + self.assertFalse(os.path.exists(self.lock)) + + def test_the_tasks_ports_are_named_and_locked_until_the_unit_ends(self): + with self.driver._published("082") as env: + self.assertEqual(env, {"OSWORLD2_GUEST_PORTS": str(self.port)}) + self.assertTrue(self.held()) + self.assertEqual(len(self.driver._PORTS), 1) + self.assertFalse(self.held()) + self.assertEqual(self.driver._PORTS, []) + + def test_the_reaper_keeps_the_lock(self): + with mock.patch.object(self.driver.subprocess, "Popen") as popen, self.driver._published("082"): + held = tuple(self.driver._PORTS) + self.driver._reaper(1, self.tmp, True) + self.assertEqual(len(held), 1) + self.assertEqual(popen.call_args.kwargs["pass_fds"], held) + + def test_a_listener_on_loopback_is_refused_and_the_lock_let_go(self): + with socket.socket() as s: + s.bind(("127.0.0.1", self.port)) + s.listen() + with self.assertRaisesRegex(RuntimeError, rf"task 082 .* port {self.port} .*127\.0\.0\.1"): + with self.driver._published("082"): + self.fail("published while another process listens") + self.assertFalse(self.held()) + self.assertEqual(self.driver._PORTS, []) + + def test_a_listener_on_every_interface_is_refused(self): + with socket.socket() as s: + s.bind(("0.0.0.0", self.port)) + s.listen() + with self.assertRaisesRegex(RuntimeError, rf"task 082 .* port {self.port} "): + with self.driver._published("082"): + self.fail("published while another process listens") + self.assertFalse(self.held()) + + def test_a_listener_on_ipv6_loopback_is_refused(self): + try: + s = socket.socket(socket.AF_INET6) + s.bind(("::1", self.port)) + except OSError: + self.skipTest("no IPv6 loopback here") + with s: + s.listen() + with self.assertRaisesRegex(RuntimeError, r"listens on ::1"): + with self.driver._published("082"): + self.fail("published while another process listens") + + def test_a_connection_closing_on_the_port_is_no_listener(self): + with socket.socket() as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", self.port)) + server.listen() + client = socket.create_connection(("127.0.0.1", self.port)) + accepted, _ = server.accept() + accepted.close() + client.close() + with self.driver._published("082") as env: + self.assertEqual(env, {"OSWORLD2_GUEST_PORTS": str(self.port)}) + + def test_another_unit_waits_until_its_time_runs_out(self): + fd = os.open(self.lock, os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + threading.Timer(1.5, self.driver.CLOSED.set).start() + with self.assertRaisesRegex(RuntimeError, rf"time ran out while another unit's VM held port {self.port}"): + with self.driver._published("082"): + self.fail("published while another unit holds the port") + self.assertEqual(self.driver._PORTS, []) + + def test_a_fifo_in_the_locks_place_is_refused_at_once(self): + os.mkfifo(self.lock) + with self.assertRaisesRegex(RuntimeError, "cannot be opened"): + with self.driver._published("082"): + self.fail("published through a FIFO") + self.assertEqual(self.driver._PORTS, []) + + def test_the_lock_is_readable_by_every_user_whatever_the_umask(self): + mask = os.umask(0o077) + try: + with self.driver._published("082"): + pass + finally: + os.umask(mask) + self.assertEqual(os.stat(self.lock).st_mode & 0o777, 0o644) + + def test_a_link_in_the_locks_place_is_refused(self): + os.symlink(os.path.join(self.tmp, "elsewhere"), self.lock) + with self.assertRaisesRegex(RuntimeError, "cannot be opened"): + with self.driver._published("082"): + self.fail("published through a link") + self.assertFalse(os.path.exists(os.path.join(self.tmp, "elsewhere"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_judge.py b/app/tests/test_osworld2_judge.py new file mode 100644 index 0000000..e31d003 --- /dev/null +++ b/app/tests/test_osworld2_judge.py @@ -0,0 +1,657 @@ +"""OSWorld2's LLM grader, simulated user and judge route, and its pinned dataset lists. +With NULLIUS_OSWORLD2_UPSTREAM set, the lists are also checked against the task classes, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import providers +from nullius.standard import definitions, relay, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +OFFLINE36 = ("001 002 004 010 012 022 029 030 040 042 044 045 047 048 049 051 053 054 059 061 063 064 066 067 " + "076 080 084 085 088 094 103 104 105 106 107 108").split() +MORE27 = "003 006 009 011 014 017 023 024 025 027 032 033 034 071 074 078 083 086 087 089 090 092 093 095 096 097 099".split() +PROXY = "036 037 050 055 056 062 075 098".split() +CATCHING = "003 008 019 025 031 041 077 079 082 090 092".split() +SONNET = "anthropic/claude-sonnet-4.6" + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_judge", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def bench(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f) + + +class Started(Exception): + pass + + +class Driver(unittest.TestCase): + def setUp(self): + self.job = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.job, ignore_errors=True) + trial = os.path.join(self.job, "u1") + self.tmp = os.path.join(trial, "workspace") + self.relayed = os.path.join(trial, ".work", "relay", "u1-judge.jsonl") + os.makedirs(os.path.dirname(self.relayed)) + os.makedirs(self.tmp) + self.driver = load_driver(self.tmp) + self.judge = {"model": SONNET, "base_url": "http://127.0.0.1:9/u1-judge/v1", "key": relay.PLACEHOLDER_KEY} + self.vms, self.envs = [], [] + + def vm(dir, domain, eid): + self.vms.append(eid) + return os.path.join(dir, "vm.qcow2") + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + for name, fake in (("_vm", vm), ("_run", run), ("_task", lambda *_: {"instruction": "", "proxy": False}), + ("_assets", lambda *a: "assets")): + p = mock.patch.object(self.driver, name, fake) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid, **kw): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", + relay.PLACEHOLDER_KEY, **kw) + + def nop(self, eid, **kw): + return self.driver.score(f"tasks__{eid}", self.tmp, **kw) + + def test_the_grader_gets_the_judge_route_in_the_original_and_the_null_control(self): + for start in (self.answer, self.nop): + with self.subTest(start.__name__), self.assertRaises(Started): + start("003", judge=self.judge) + env = self.envs[-1] + self.assertEqual({k: v for k, v in env.items() if k.startswith("OSWORLD_EVAL_MODEL_")}, + {"OSWORLD_EVAL_MODEL_PROVIDER": "anthropic", "OSWORLD_EVAL_MODEL_NAME": SONNET, + "OSWORLD_EVAL_MODEL_BASE_URL": "http://127.0.0.1:9/u1-judge", + "OSWORLD_EVAL_MODEL_API_KEY": relay.PLACEHOLDER_KEY}) + + def test_only_a_trailing_v1_leaves_the_route(self): + with self.assertRaises(Started): + self.nop("001", judge=dict(self.judge, base_url="http://h:1/v1-judge/v1")) + self.assertEqual(self.envs[-1]["OSWORLD_EVAL_MODEL_BASE_URL"], "http://h:1/v1-judge") + + def test_the_simulated_user_asks_sonnet_on_the_units_own_route(self): + with self.assertRaises(Started): + self.answer("024", judge=self.judge) + env = self.envs[-1] + self.assertEqual({k: v for k, v in env.items() if k.startswith("OSWORLD_USER_SIM_")}, + {"OSWORLD_USER_SIM_PROVIDER": "anthropic", "OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6", + "OSWORLD_USER_SIM_BASE_URL": "http://127.0.0.1:9/u1", + "OSWORLD_USER_SIM_API_KEY": relay.PLACEHOLDER_KEY}) + self.assertEqual(env["ANTHROPIC_BASE_URL"], "http://127.0.0.1:9/u1") + + def test_a_graded_task_with_no_judge_route_is_refused_before_its_vm(self): + for start in (self.answer, self.nop): + for eid in ("003", "093"): + with self.subTest(f"{start.__name__} {eid}"), self.assertRaises(RuntimeError) as raised: + start(eid) + self.assertIn(f"task {eid} is scored by the LLM grader", str(raised.exception)) + self.assertEqual(self.vms, []) + + def test_a_bad_judge_route_is_refused_before_the_vm(self): + with self.assertRaises(RuntimeError) as raised: + self.nop("001", judge={"model": SONNET, "base_url": ""}) + self.assertIn("judge route needs", str(raised.exception)) + self.assertEqual(self.vms, []) + + def test_an_ungraded_task_runs_with_no_judge_route_and_no_grader_settings(self): + for start in (self.answer, self.nop): + with self.subTest(start.__name__), self.assertRaises(Started): + start("001") + self.assertFalse([k for k in self.envs[-1] if k.startswith("OSWORLD_EVAL_MODEL_")]) + self.assertEqual(self.vms, ["001", "001"]) + + def test_a_simulated_user_with_no_route_is_refused_before_its_vm(self): + agents = json.loads(json.dumps(self.driver.AGENTS)) + for k in [k for k in agents["agents"]["m3"]["env"] if k.startswith("OSWORLD_USER_SIM_")]: + del agents["agents"]["m3"]["env"][k] + with mock.patch.object(self.driver, "AGENTS", agents): + with self.assertRaises(RuntimeError) as raised: + self.answer("095", judge=self.judge) + self.assertIn("task 095 has a simulated user", str(raised.exception)) + self.assertEqual(self.vms, []) + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + + def result(self, where, value="0.0"): + os.makedirs(where, exist_ok=True) + with open(os.path.join(where, "result.txt"), "w") as f: + f.write(value) + + def log(self, path, *lines): + with open(path, "a") as f: + f.write("".join(ln + "\n" for ln in lines)) + + def test_a_failed_grader_is_the_original_scores_error_not_a_0(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 0.0) + failed = "\x1b[1;33m[2026-11-06 01:02:03 \x1b[31mERROR \x1b[32mllm_metrics/388-Worker\x1b[1;33m] \x1b[0m" \ + "Error in compare_text_with_llm: Error code: 402 - insufficient credits" + for log in (os.path.join(where, "eval.log"), os.path.join(self.tmp, "run_multienv_m3.log")): + with self.subTest(os.path.basename(log)): + self.log(log, "[2026-11-06 01:02:00 INFO desktop_env/1] evaluating", failed) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + said = str(raised.exception) + self.assertIn("the LLM grader failed in compare_text_with_llm (HTTP 402), and the upstream", said) + self.assertIn("see the unit's judge route", said) + self.assertNotIn("insufficient credits", said) + self.assertNotIn("\x1b", said) + os.remove(log) + + def test_an_earlier_runs_results_are_moved_aside_before_the_runner(self): + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001"), "1.0") + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + self.assertFalse(os.path.lexists(os.path.join(self.tmp, "results"))) + (earlier,) = glob.glob(os.path.join(self.tmp, "results.*")) + self.assertTrue(glob.glob(os.path.join(earlier, "**", "result.txt"), recursive=True)) + + def test_an_earlier_runs_logs_are_moved_aside_and_its_grader_failure_is_not_this_runs(self): + failed = "[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: Error code: 402 - x" + for name in ("run_multienv_m3.log", "nop.log", "setup.log"): + self.log(os.path.join(self.tmp, name), failed) + with self.assertRaises(Started): + self.answer("003", judge=self.judge) + self.assertEqual(sorted(os.path.basename(p).rsplit(".", 1)[0] for p in glob.glob(os.path.join(self.tmp, "*.log.*"))), + ["nop.log", "run_multienv_m3.log", "setup.log"]) + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003"), "1.0") + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 1.0) + self.log(os.path.join(self.tmp, "nop.log"), failed) + with contextlib.suppress(RuntimeError): + self.driver.setup("tasks__003", self.tmp) + nop = os.path.join(self.tmp, "results", "nop", "tasks", "003") + with mock.patch.object(self.driver, "_run", lambda *a, **kw: self.result(nop, "1.0")): + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 1.0) + + def test_a_unit_an_earlier_run_or_its_reaper_still_holds_is_not_begun(self): + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, os, sys, time\n" + "fd = os.open(sys.argv[1], os.O_RDONLY | os.O_CREAT)\nfcntl.flock(fd, fcntl.LOCK_EX)\n" + "print(flush=True)\ntime.sleep(30)", os.path.join(self.tmp, "unit.lck")], + stdout=subprocess.PIPE) + self.addCleanup(holder.kill) + holder.stdout.readline() + with mock.patch.object(self.driver, "HELD_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.answer("001", judge=self.judge) + self.assertIn("an earlier run of this unit, or its reaper, still holds its folder", str(raised.exception)) + holder.kill() + holder.wait() + with self.assertRaises(Started): + self.answer("001", judge=self.judge) + self.assertEqual(self.driver._OWN, []) + self.driver._fresh(self.tmp) + with mock.patch.object(self.driver.subprocess, "Popen") as started: + self.driver._arm(os.getpid(), self.tmp, True) + os.close(self.driver._ARMED.pop(os.getpid())) + self.assertIn(self.driver._OWN[0], started.call_args.kwargs["pass_fds"]) + self.result(os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001"), "1.0") + self.driver.score("tasks__001", self.tmp, setup="original") + self.assertEqual(self.driver._OWN, []) + + def test_a_result_outside_0_to_1_is_no_score(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + for value in ("nan", "inf", "-inf", "-0.5", "1.5"): + with self.subTest(value): + self.result(where, value) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__001", self.tmp, setup="original") + self.assertIn("not a score from 0 to 1", str(raised.exception)) + for value in ("0", "0.25", "1.0", "1.0000000000000002"): + self.result(where, value) + self.assertEqual(self.driver.score("tasks__001", self.tmp, setup="original")["reward"], float(value)) + + def test_a_failed_graders_message_says_its_status_and_class_never_its_words(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + words = "the task's own words" + records = {"Error code: 429 - " + words: "HTTP 429", + "RateLimitError: " + words: "RateLimitError", + "anthropic.APIStatusError: status code 529 " + words: "anthropic.APIStatusError, HTTP 529", + "openai.BadRequestError: " + words: "openai.BadRequestError", + "anthropic._exceptions.OverloadedError: " + words: "anthropic.OverloadedError", + "httpx.ReadTimeout " + words: "httpx.ReadTimeout", + "TimeoutError: " + words: "TimeoutError", + words + " 500 times": "no HTTP status or known error class in its record"} + log = os.path.join(where, "eval.log") + for record, said in records.items(): + with self.subTest(said): + self.log(log, f"[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: {record}") + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + self.assertIn(f"compare_text_with_llm ({said})", str(raised.exception)) + self.assertNotIn(words, str(raised.exception)) + os.remove(log) + + def test_a_word_shaped_like_a_class_is_never_said(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + log = os.path.join(where, "eval.log") + for record in ("SyntheticPrivateDocumentError: the rest", "PrivateLedgerException the rest", + "anthropic.SyntheticPrivateError: x", "SyntheticPrivate.RateLimitError: x", + "secret.module.ValueError: x", "RateLimitErrorOfTheTask: x", "rate_limit_error: x", + "anthropic.RateLimitErrorXYZ: x"): + with self.subTest(record): + self.log(log, f"[2026-11-06 01:02:03 ERROR llm_metrics/388-Worker] Error in compare_text_with_llm: {record}") + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + said = str(raised.exception) + self.assertIn("compare_text_with_llm (no HTTP status or known error class in its record)", said) + for word in ("Synthetic", "Private", "secret", "XYZ", "OfTheTask", "rate_limit"): + self.assertNotIn(word, said) + os.remove(log) + + def test_a_refusing_endpoints_error_text_is_not_said(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + self.calls({"status": 400, "request": {"n": 1}, "error": "status 400: the task's own words"}) + with self.assertRaises(RuntimeError) as raised: + self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + self.assertIn("answered 400", str(raised.exception)) + self.assertNotIn("own words", str(raised.exception)) + os.remove(self.relayed) + + def test_the_marks_words_outside_the_graders_error_record_change_nothing(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where, "1.0") + words = "Error in compare_text_with_llm: Error code: 402" + self.log(os.path.join(self.tmp, "run_multienv_m3.log"), + f"\x1b[1;33m[2026-11-06 01:02:03,004 \x1b[31mINFO \x1b[32mrun_multienv_m3/512-Worker\x1b[1;33m] \x1b[0m" + f"Step 3: pyautogui.typewrite('{words}')", + f"[2026-11-06 01:02:03,004 ERROR lib_run_single/88-Worker] step failed: {words}", + f"[2026-11-06 01:02:03,004 ERROR run_multienv_m3/600-Worker] {words}", + words) + self.log(os.path.join(where, "eval.log"), f"[2026-11-06 01:02:03,004 INFO llm_metrics/40-Worker] {words}") + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 1.0) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + self.log(log, f"[2026-11-06 01:02:04,005 INFO desktopenv.env] user said: {words}", + f"[2026-11-06 01:02:04,005 ERROR desktopenv.experiment] {words}") + + with mock.patch.object(self.driver, "_run", run): + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + + def calls(self, *calls): + with open(self.relayed, "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls) + "not json\n") + + def test_a_grader_endpoints_failure_no_retry_recovered_refuses_the_score(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + + asked, other = {"n_messages": 1, "last": ["grade this"], "sha256": "a"}, {"n_messages": 1, "last": ["grade that"], "sha256": "b"} + undigested = {"n_messages": 1, "last": ["grade this"]} + ok = lambda req: {"status": 200, "request": req, "reply": {"text": "YES"}} + bad = lambda status, req: {"status": status, "request": req, "error": f"status {status}: credits"} + refused = {"402 then another call's 200": [bad(402, asked), ok(other)], + "429 three times": [bad(429, asked), bad(429, asked), bad(429, asked)], + "500 last": [ok(other), bad(500, asked)], + "400 then 200 on another request": [bad(400, other), ok(asked)], + "402, another call, then 200 to its words": [bad(402, asked), ok(other), ok(asked)], + "no request recorded": [{"status": 500, "error": "x"}, {"status": 200}], + "a late 200 recorded after the last failure": [dict(bad(500, asked), t0=2), dict(ok(asked), t0=1)], + "no digest of the request": [bad(429, undigested), ok(undigested)], + "a failure with no start time": [dict(ok(asked), t0=1), bad(429, asked)]} + recovered = {"none": [], "429 then 200": [bad(429, asked), ok(asked)], + "503, 503, 200": [bad(503, asked), bad(503, asked), ok(asked), ok(other)], + "an abandoned 502 recorded after its retry's 200": [dict(ok(asked), t0=2), dict(bad(502, asked), t0=1)]} + with mock.patch.object(self.driver, "_run", run): + for name, calls in refused.items(): + self.calls(*calls) + for setup, start in (("original", lambda: self.driver.score("tasks__003", self.tmp, setup="original", + judge=self.judge)), + ("nop", lambda: self.nop("003", judge=self.judge))): + with self.subTest(f"{name} {setup}"), self.assertRaises(RuntimeError) as raised: + start() + self.assertIn("the LLM grader's endpoint answered", str(raised.exception)) + self.assertIsInstance(raised.exception, self.driver.VerifierError) + self.assertIn("see the unit's judge route", str(raised.exception)) + for name, calls in recovered.items(): + self.calls(*calls) + with self.subTest(name): + self.assertEqual(self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge)["reward"], 0.0) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + os.remove(self.relayed) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + self.calls(bad(402, asked)) + self.assertEqual(self.nop("001")["reward"], 0.0) + self.assertEqual(self.nop("001", judge=dict(self.judge, base_url="http://h:1/u1/v1"))["reward"], 0.0) + + def test_the_agents_file_refused_keeps_the_upstreams_score_by_the_images_sent_never_the_words(self): + where = os.path.join(self.tmp, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "003") + self.result(where) + asked = {"n_messages": 1, "sha256": "a"} + whole, broken = {"type": "image/png", "is": "image/png", "bytes": 4000}, {"type": "image/png", "is": None, "bytes": 4000} + big = dict(whole, bytes=self.driver.IMAGE_BYTES + 1) + said = "messages.0.content.1.image.source.base64.data: could not process image, too large" + bad = lambda status, *images: dict({"status": status, "request": asked, "error": f"HTTP {status}: {said}"}, + **({"images": list(images)} if images else {})) + file, size = bad(400, whole, broken), bad(413, whole, big) + record = "[2026-11-06 01:02:04 ERROR desktopenv.env] Error in compare_images_with_llm: Error code: {} - x" + score = lambda: self.driver.score("tasks__003", self.tmp, setup="original", judge=self.judge) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + + def nop(): + try: + return self.nop("003", judge=self.judge) + finally: + shutil.rmtree(os.path.join(self.tmp, "results", "nop"), ignore_errors=True) + + with mock.patch.object(self.driver, "_run", run): + for calls, refused in (([file], ["agent file"]), ([size, file], ["agent file", "agent file size"]), + ([bad(422, {"type": "image/bmp", "is": "image/bmp", "bytes": 10})], ["agent file"])): + self.calls(*calls) + with open(os.path.join(where, "eval.log"), "w") as f: + f.write("".join(record.format(c["status"]) + "\n" for c in calls)) + with self.subTest(refused=refused): + self.assertEqual((score()["reward"], score()["grader_refused"]), (0.0, refused)) + self.assertEqual(dict(nop(), detail=None), {"reward": 0.0, "detail": None, "grader_refused": refused}) + os.remove(os.path.join(where, "eval.log")) + for name, calls in (("the provider's words alone", [bad(400)]), ("whole images", [bad(400, whole, whole)]), + ("a whole image said to be another type", [bad(400, dict(whole, type="image/jpeg"))]), + ("a 413 of whole images", [bad(413, whole)]), ("a 413 of a broken one", [bad(413, broken)]), + ("a 401", [bad(401, broken)]), ("with a 429", [file, bad(429)])): + self.calls(*calls) + for setup, start in (("original", score), ("nop", nop)): + with self.subTest(f"{name} {setup}"), self.assertRaises(self.driver.VerifierError) as raised: + start() + self.assertIn("the LLM grader's endpoint answered", str(raised.exception)) + self.calls(file) + for status in (429, 400): + with open(os.path.join(where, "eval.log"), "w") as f: + f.write(record.format(400) + "\n" + record.format(status) + "\n") + with self.subTest(status=status), self.assertRaises(self.driver.VerifierError) as raised: + score() + self.assertIn(f"compare_images_with_llm (HTTP {status})", str(raised.exception)) + os.remove(os.path.join(where, "eval.log")) + other = os.path.join(self.job, "u2", "workspace") + self.result(os.path.join(other, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "005")) + shutil.copytree(os.path.dirname(os.path.dirname(self.relayed)), os.path.join(self.job, "u2", ".work")) + for eid, dir in (("005", other), ("003", self.tmp), ("005", other)): + with self.subTest(eid=eid): + self.assertEqual(self.driver.score(f"tasks__{eid}", dir, setup="original", judge=self.judge)["grader_refused"], + ["agent file"]) + + def test_a_failed_grader_is_the_null_controls_error_not_a_0(self): + marks = list(self.driver.GRADER_FAILED) + + def run(argv, cwd, env, log, *a, **kw): + self.result(argv[-1]) + self.log(log, "[2026-11-06 01:02:03 INFO desktopenv.env] evaluating", + *([f"[2026-11-06 01:02:04 ERROR desktopenv.env] {marks.pop(0)}Error code: 429"] if marks else [])) + + with mock.patch.object(self.driver, "_run", run): + for mark in list(marks): + with self.subTest(mark), self.assertRaises(RuntimeError) as raised: + self.nop("003", judge=self.judge) + self.assertIn(f"{mark[len('Error in '):-2]} (HTTP 429)", str(raised.exception)) + os.remove(os.path.join(self.tmp, "nop.log")) + self.assertEqual(self.nop("003", judge=self.judge)["reward"], 0.0) + + def test_the_lists(self): + lists = self.driver.LISTS + self.assertEqual(lists["offline36"], OFFLINE36) + self.assertEqual(lists["no_services63"], sorted(OFFLINE36 + MORE27)) + self.assertEqual(len(set(lists["no_services63"])), 63) + self.assertFalse(set(lists["no_services63"]) & set(PROXY)) + self.assertEqual(len(self.driver.JUDGED), 21) + self.assertEqual(len(self.driver.ASKS_USER), 6) + self.assertFalse((self.driver.JUDGED | self.driver.ASKS_USER) & set(OFFLINE36)) + with mock.patch.dict(os.environ, {"NULLIUS_DATASET": "", "NULLIUS_DATASET_ID": "no_services63"}): + self.assertEqual(self.driver.units()["units"], [f"tasks__{e}" for e in sorted(OFFLINE36 + MORE27)]) + counted = {d["id"]: d["units"] for d in bench()["datasets"]} + self.assertEqual((counted["offline36"], counted["no_services63"]), (36, 63)) + + +class Definition(unittest.TestCase): + def test_the_judge_and_the_simulated_user_are_sonnet_as_openrouter_names_it(self): + b = bench() + self.assertEqual(definitions.problems(b, "benchmark"), []) + self.assertEqual(providers.served_as(b["score"]["judge"]["model"], "openrouter"), SONNET) + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + asked = json.load(f)["agents"]["m3"]["env"]["OSWORLD_USER_SIM_MODEL"] + other, why = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.assertEqual(why, "") + self.assertEqual(other["sent"], "minimax/minimax-m3") + self.assertEqual(other["model"]["side_models"], [f"{asked}={SONNET}"]) + + +class Provider(BaseHTTPRequestHandler): + seen = [] + status = {} + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + Provider.seen.append((self.path, body["model"])) + code = int(self.headers.get("X-Status") or 200) + out = (json.dumps({"type": "message", "role": "assistant", "content": [{"type": "text", "text": "ok"}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) if code == 200 + else json.dumps({"error": {"message": f"status {code}"}})).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + +class Routes(unittest.TestCase): + """The unit's relay as an Original run of OSWorld2 on OpenRouter starts it, before a fake provider.""" + + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + b = bench() + other, _ = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.judge_model = providers.served_as(b["score"]["judge"]["model"], "openrouter") + self.trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.trial, ignore_errors=True) + self.relay = relay.Relay(f"http://127.0.0.1:{self.up.server_address[1]}/api/v1", "sk-real", + os.path.join(self.trial, ".work", "relay"), + model=other["sent"], side_models=other["model"]["side_models"], + key_header=run.PROVIDERS["openrouter"].get("key_header", "authorization")).start() + self.addCleanup(self.relay.stop) + self.judge_url = self.relay.url("u1-judge", model=self.judge_model) + + def post(self, url, model, status=200, words=""): + messages = [{"role": "user", "content": words}] if words else [] + req = urllib.request.Request(url + "/messages", data=json.dumps({"model": model, "max_tokens": 8, + "messages": messages}).encode(), + headers={"Content-Type": "application/json", "x-api-key": relay.PLACEHOLDER_KEY, + "X-Status": str(status), + "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status + except urllib.error.HTTPError as e: + e.close() + return e.code + + def test_sonnet_is_never_sent_as_m3_and_the_graders_calls_are_not_the_agents(self): + unit = self.relay.url("u1") + self.post(unit, "MiniMax-M3") + self.post(unit, "claude-sonnet-4-6") + self.post(self.judge_url, "claude-sonnet-4-6") + self.assertEqual([m for _, m in Provider.seen[-3:]], ["minimax/minimax-m3", SONNET, SONNET]) + self.assertTrue(all(p == "/api/v1/messages" for p, _ in Provider.seen[-3:])) + agent, user = self.relay.calls("u1") + self.assertEqual((agent["model"], agent.get("side")), ("minimax/minimax-m3", None)) + self.assertEqual((user["model"], user["model_asked"], user.get("side")), (SONNET, "claude-sonnet-4-6", True)) + self.assertEqual([c["model"] for c in self.relay.calls("u1-judge")], [SONNET]) + + def test_the_providers_refusals_are_the_judge_routes_failure(self): + for status, kind in ((402, "ApiError"), (429, "RateLimitError"), (500, "ApiError"), (503, "ApiError")): + with self.subTest(status): + self.assertEqual(self.post(self.judge_url, SONNET, status), status) + self.assertEqual(relay.failure(self.relay.calls("u1-judge"))[0], kind) + self.assertEqual(self.post(self.judge_url, SONNET), 200) + self.assertIsNone(relay.failure(self.relay.calls("u1-judge"))) + self.assertEqual(self.relay.calls("u1"), []) + + def test_the_driver_reads_the_relays_own_record_of_the_judge_route(self): + """The driver reads the relay's own record of the judge route, as run.Run writes it.""" + other, _ = definitions.setup(bench(), "original", "MiniMax-M3", "openrouter") + unit = types.SimpleNamespace(base=f"http://127.0.0.1:{self.up.server_address[1]}/api/v1", key="sk-real", call_seconds=relay.TIMEOUT, + other=other, provider="openrouter", backend="local", model=other["sent"], + judge_model=self.judge_model, + serving=run.pins_of(bench(), other, "openrouter", self.judge_model), ledger_file=lambda: None) + trial = os.path.join(self.trial, "unit") + rl = run.Run.unit_relay(unit, "u1", trial, {}) + self.addCleanup(rl.stop) + judge = run.Run.judge(unit, rl, "u1")["judge"] + driver = load_driver(self.trial) + ws = os.path.join(trial, "workspace") + driver._grader_refused(ws, judge) + self.post(judge["base_url"], SONNET, 429, "first") + self.post(judge["base_url"], SONNET, 200, "first") + self.post(judge["base_url"], SONNET, 200, "second") + driver._grader_refused(ws, judge) + self.assertEqual([c["status"] for c in rl.calls("u1-judge")], [429, 200, 200]) + self.assertEqual({c["request"]["routing"]["order"][0] for c in rl.calls("u1-judge")}, {"anthropic"}) + self.post(judge["base_url"], SONNET, 402, "third") + self.post(judge["base_url"], SONNET, 200, "fourth") + with self.assertRaises(RuntimeError) as raised: + driver._grader_refused(ws, judge) + self.assertIn("answered 402, and the task was scored without it", str(raised.exception)) + self.assertNotIn("status 402", str(raised.exception)) + + def test_a_retry_is_the_whole_request_again_not_only_its_words(self): + shot = lambda data: [{"type": "text", "text": "the same words"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": data}}] + self.post(self.judge_url, SONNET, 429, shot("QQ==")) + self.post(self.judge_url, SONNET, 200, shot("Qg==")) + first, second = self.relay.calls("u1-judge") + self.assertNotEqual(first["request"], second["request"]) + with self.assertRaises(RuntimeError): + load_driver(self.trial)._grader_refused(os.path.join(self.trial, "workspace"), {"base_url": self.judge_url}) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +def _metrics(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + path = os.path.join(up, "desktop_env", "evaluators", "metrics", "llm_metrics.py") + return path if up and os.path.isfile(path) else "" + + +@unittest.skipUnless(_metrics(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream") +class Metrics(unittest.TestCase): + def test_each_mark_is_what_the_upstreams_llm_metrics_log(self): + with open(_metrics(), encoding="utf-8") as f: + source = f.read() + driver = load_driver(os.environ["NULLIUS_OSWORLD2_UPSTREAM"]) + for mark in driver.GRADER_FAILED: + self.assertEqual(source.count(f'logger.error("{mark}%s", exc)'), 1, mark) + self.assertEqual(source.count('logger.error("Error in '), len(driver.GRADER_FAILED)) + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + """What each task class asks for, read from its syntax tree.""" + + def test_the_drivers_lists_are_the_task_classes(self): + graded, users, hosted, proxied = set(), set(), set(), set() + for p in _task_classes(): + n = os.path.basename(p)[5:-3] + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + if isinstance(node, ast.Call): + f = node.func + name = f.id if isinstance(f, ast.Name) else f.attr if isinstance(f, ast.Attribute) else "" + if name in ("generate_text", "generate_chat") or name.endswith("_with_llm"): + graded.add(n) + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [a.name for a in node.names] if isinstance(node, ast.Import) else []) + if any("llm_metrics" in m or "model_client" in m for m in modules): + graded.add(n) + if any(m.endswith(("controllers.website", "controllers.gitlab")) for m in modules) or \ + (isinstance(node, ast.ImportFrom) and (node.module or "").endswith("controllers") + and {a.name for a in node.names} & {"website", "gitlab"}): + hosted.add(n) + if isinstance(node, ast.ClassDef): + for st in node.body: + targets = st.targets if isinstance(st, ast.Assign) else [st.target] if isinstance(st, ast.AnnAssign) else [] + names = {t.id for t in targets if isinstance(t, ast.Name)} + value = getattr(st, "value", None) + if "proxy" in names and isinstance(value, ast.Constant) and value.value is True: + proxied.add(n) + if "user_simulator" in names and isinstance(value, ast.Dict): + if any(isinstance(k, ast.Constant) and k.value == "type" and isinstance(v, ast.Constant) + and v.value == "llm" for k, v in zip(value.keys, value.values)): + users.add(n) + driver = load_driver(os.environ["NULLIUS_OSWORLD2_UPSTREAM"]) + self.assertEqual(sorted(graded), sorted(driver.JUDGED)) + self.assertEqual(sorted(users), sorted(driver.ASKS_USER)) + self.assertEqual(sorted(proxied), PROXY) + self.assertFalse(set(driver.LISTS["no_services63"]) & (hosted | proxied)) + self.assertEqual(len(_task_classes()) - len(hosted | proxied), 63) + + def test_the_graded_tasks_that_catch_a_grader_error_themselves(self): + """The graded tasks that catch a grader error themselves; pinned, so a change shows.""" + catching = set() + for p in _task_classes(): + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + if not isinstance(node, ast.Try) or all(any(isinstance(x, ast.Raise) for x in ast.walk(h)) + for h in node.handlers): + continue + if any(isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute)) + and (c.func.id if isinstance(c.func, ast.Name) else c.func.attr) in ("generate_text", "generate_chat") + for part in node.body for c in ast.walk(part)): + catching.add(os.path.basename(p)[5:-3]) + self.assertEqual(sorted(catching), CATCHING) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_mail_profiles.py b/app/tests/test_osworld2_mail_profiles.py new file mode 100644 index 0000000..f4d5ebe --- /dev/null +++ b/app/tests/test_osworld2_mail_profiles.py @@ -0,0 +1,136 @@ +"""OSWorld2's mail tasks get their profiles signed out of the live mailboxes their assets ship, and started offline.""" +from __future__ import annotations + +import contextlib +import io +import os +import tarfile +import tempfile +import unittest +import zlib +from unittest import mock + +from tests.test_osworld2_driver import load + +PROFILE = ".thunderbird/abc.default-release/" +PREFS = ('user_pref("mail.server.server1.hostname", "mail.example.com");\n' + 'user_pref("mail.server.server1.login_at_startup", true);\n' + 'user_pref("mail.server.server1.check_new_mail", true);\n' + 'user_pref("mail.server.server1.download_on_biff", true);\n' + ' user_pref("mail.server.server2.check_new_mail",true);\r\n' + 'user_pref("offline.startup_state", 2);\n') +SIGNED_IN = ("logins.json", "logins-backup.json", "signons.sqlite", "cookies.sqlite", "cookies.sqlite-journal", "user.js", + "storage/default/https+++login.example.com/ls/data.sqlite") + + +def archive(path, files): + with tarfile.open(path, "w:gz") as t: + for name, data in files.items(): + info = tarfile.TarInfo(name) + info.size, info.mtime = len(data), 1700000000 + t.addfile(info, io.BytesIO(data)) + + +def members(path): + with tarfile.open(path) as t: + return {m.name: t.extractfile(m).read() for m in t if m.isfile()} + + +class SignedOut(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + up = os.path.join(self.tmp.name, "up") + assets = os.path.join(up, "cache", "osworld_v2_assets") + for d in ("task_006", "task_007", "shared"): + os.makedirs(os.path.join(assets, d)) + self.files = {PROFILE + "prefs.js": PREFS.encode(), PROFILE + "key4.db": b"k", + PROFILE + "Mail/host/Inbox": b"From a\n\nbody\n", PROFILE + "storage/permanent/chrome/idb/x": b"i", + **{PROFILE + n: b"fake" for n in SIGNED_IN}} + archive(os.path.join(assets, "task_006", "thunderbird-profile.tar.gz"), self.files) + with open(os.path.join(assets, "task_006", "sheet.xlsx"), "wb") as f: + f.write(b"sheet") + self.driver = load(up) + self.unit = os.path.join(self.tmp.name, "unit") + os.makedirs(self.unit) + + def tearDown(self): + self.tmp.cleanup() + + def test_other_tasks_keep_the_assets_folder(self): + self.assertEqual(self.driver._assets(self.unit, "007"), self.driver.ASSETS) + self.assertFalse(os.path.exists(os.path.join(self.unit, "assets"))) + + def test_the_profile_loses_its_sign_ins_and_keeps_its_mail(self): + root = self.driver._assets(self.unit, "006") + got = members(os.path.join(root, "task_006", "thunderbird-profile.tar.gz")) + self.assertEqual(sorted(got), sorted(n for n in self.files if n[len(PROFILE):] not in SIGNED_IN)) + self.assertEqual(got[PROFILE + "Mail/host/Inbox"], self.files[PROFILE + "Mail/host/Inbox"]) + prefs = got[PROFILE + "prefs.js"].decode() + self.assertNotIn("true", prefs) + self.assertIn('"mail.server.server1.hostname", "mail.example.com"', prefs) + self.assertTrue(prefs.endswith('user_pref("offline.autoDetect", false);\n' + 'user_pref("offline.startup_state", 3);\n')) + + def test_the_rest_of_the_assets_are_the_originals(self): + root = self.driver._assets(self.unit, "006") + for rel in ("shared", "task_007", os.path.join("task_006", "sheet.xlsx")): + self.assertEqual(os.path.realpath(os.path.join(root, rel)), + os.path.realpath(os.path.join(self.driver.ASSETS, rel))) + self.assertNotIn("thunderbird-profile.tar.gz.part", os.listdir(os.path.join(root, "task_006"))) + + def test_the_copy_is_the_same_each_time(self): + root = self.driver._assets(self.unit, "006") + with open(os.path.join(root, "task_006", "thunderbird-profile.tar.gz"), "rb") as f: + first = f.read() + self.driver._assets(self.unit, "006") + with open(os.path.join(root, "task_006", "thunderbird-profile.tar.gz"), "rb") as f: + self.assertEqual(f.read(), first) + + def test_a_missing_profile_is_refused(self): + os.remove(os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz")) + with self.assertRaisesRegex(RuntimeError, "no thunderbird-profile.tar.gz"): + self.driver._assets(self.unit, "006") + + def test_a_hard_link_is_refused(self): + path = os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz") + with tarfile.open(path, "w:gz") as t: + link = tarfile.TarInfo(PROFILE + "prefs.js") + link.type, link.linkname = tarfile.LNKTYPE, PROFILE + "key4.db" + t.addfile(link) + with self.assertRaisesRegex(RuntimeError, "does not rewrite"): + self.driver._assets(self.unit, "006") + + def test_a_broken_profile_archive_is_a_setup_error(self): + path = os.path.join(self.driver.ASSETS, "task_006", "thunderbird-profile.tar.gz") + with open(path, "rb") as f: + whole = f.read() + patches = {"_vm": lambda *a: "vm", "_task": lambda *_: {"instruction": "", "proxy": False}} + for name, data, fault in (("not an archive", b"x" * 600, None), ("cut short", whole[:len(whole) // 2], None), + ("bad deflate data", whole, zlib.error("invalid stored block lengths"))): + with open(path, "wb") as f: + f.write(data) + with self.subTest(name), mock.patch.multiple(self.driver, **patches), \ + mock.patch.object(self.driver, "_signed_out", side_effect=fault) if fault else contextlib.nullcontext(), \ + self.assertRaises(self.driver.EnvironmentSetupError): + self.driver.score("tasks__006", self.unit) + + def test_the_null_control_and_the_original_get_the_copy(self): + envs = [] + + def run(argv, cwd, env, *a, **kw): + envs.append(env) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", run), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}): + for start in (lambda: self.driver.score("tasks__006", self.unit), + lambda: self.driver.answer("tasks__006", self.unit, "original", "MiniMax-M3", + "http://127.0.0.1:9/v1", "k")): + with self.assertRaises(KeyboardInterrupt): + start() + self.assertEqual([e["OSWORLD_FILE_BASE_URL"] for e in envs], [os.path.join(self.unit, "assets")] * 2) + self.assertEqual(self.driver._env("007")["OSWORLD_FILE_BASE_URL"], self.driver.ASSETS) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_muse.py b/app/tests/test_osworld2_muse.py new file mode 100644 index 0000000..839e1c8 --- /dev/null +++ b/app/tests/test_osworld2_muse.py @@ -0,0 +1,396 @@ +"""OSWorld2's Muse Spark agent beside M3: its arguments, its pinned relay route, its tool calls, its call timeout. +With NULLIUS_OSWORLD2_UPSTREAM set, every flag is checked against the runner's parser, read as a syntax tree.""" +from __future__ import annotations + +import ast +import importlib.util +import json +import os +import shutil +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius.standard import container, definitions, job, jsonl, recipe, relay, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +MUSE, SENT, SONNET = "muse-spark-1.3", "meta/muse-spark-1.3", "anthropic/claude-sonnet-4.6" +UNIT = "http://127.0.0.1:9/u1/v1" + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_muse", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def bench(): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return json.load(f) + + +class Started(Exception): + pass + + +class Driver(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.vms, self.runs = [], [] + + def vm(dir, domain, eid): + self.vms.append(eid) + return os.path.join(dir, "vm.qcow2") + + def started(argv, cwd, env, *a, **kw): + self.runs.append((argv, env)) + raise Started() + + for name, fake in (("_vm", vm), ("_run", started), ("_task", lambda *_: {"instruction": "", "proxy": False})): + p = mock.patch.object(self.driver, name, fake) + p.start() + self.addCleanup(p.stop) + + def answer(self, agent, model, eid="045", **options): + env = {"NULLIUS_OPTION_AGENT": agent, **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()}} + with mock.patch.dict(os.environ, env): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", model, UNIT, relay.PLACEHOLDER_KEY, + judge={"model": SONNET, "base_url": "http://127.0.0.1:9/u1-judge/v1", + "key": relay.PLACEHOLDER_KEY}) + + def test_muse_spark_runs_its_script_with_its_published_runs_settings_on_the_relay(self): + with self.assertRaises(Started): + self.answer("muse_spark", MUSE, max_steps="500", thinking="adaptive") + argv, env = self.runs[-1] + self.assertEqual(argv[1], os.path.join(self.tmp, "scripts", "python", "run_multienv_muse_spark.py")) + flags = {w: v for w, v in zip(argv[2:], argv[3:]) if w.startswith("--") and not v.startswith("--")} + self.assertEqual({k: flags[k] for k in ("--model", "--base_url", "--reasoning_effort", "--max_tokens", "--max_steps", + "--provider_name", "--num_envs")}, + {"--model": MUSE, "--base_url": UNIT, "--reasoning_effort": "max", "--max_tokens": "262144", + "--max_steps": "500", "--provider_name": "docker", "--num_envs": "1"}) + self.assertEqual(argv[-1], "--save_model_eval_raw_info") + self.assertNotIn("--api_key", argv) + self.assertNotIn("--truncation", argv) + self.assertEqual({k: env[k] for k in ("MUSE_SPARK_API_KEY", "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT")}, + {"MUSE_SPARK_API_KEY": relay.PLACEHOLDER_KEY, "MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1"}) + self.assertFalse([k for k in env if k.startswith("M3_")]) + + def test_the_simulated_user_and_the_grader_are_m3s(self): + envs = [] + for agent, model in (("m3", "MiniMax-M3"), ("muse_spark", MUSE)): + with self.subTest(agent), self.assertRaises(Started): + self.answer(agent, model, eid="024") + envs.append({k: v for k, v in self.runs[-1][1].items() if k.startswith(("OSWORLD_USER_SIM_", "OSWORLD_EVAL_MODEL_"))}) + self.assertEqual(envs[0], envs[1]) + self.assertEqual(envs[1]["OSWORLD_USER_SIM_BASE_URL"], "http://127.0.0.1:9/u1") + self.assertEqual(envs[1]["OSWORLD_EVAL_MODEL_NAME"], SONNET) + + def test_m3_keeps_its_thinking_option(self): + with self.assertRaises(Started): + self.answer("m3", "MiniMax-M3", thinking="adaptive") + self.assertEqual(self.runs[-1][1]["M3_THINKING_MODE"], "adaptive") + + def test_every_agent_takes_the_steps_option(self): + self.assertIsNone(definitions.option_problem(bench()["options"]["max_steps"], "300")) + for agent, model in (("m3", "MiniMax-M3"), ("muse_spark", MUSE)): + with self.subTest(agent), self.assertRaises(Started): + self.answer(agent, model, max_steps="300") + argv = self.runs[-1][0] + self.assertEqual(argv[argv.index("--max_steps") + 1], "300") + + def test_a_model_of_the_other_agent_is_refused_before_the_vm_naming_its_agent(self): + for agent, model, other in (("m3", MUSE, "muse_spark"), ("muse_spark", "MiniMax-M3", "m3")): + with self.subTest(agent), self.assertRaises(RuntimeError) as raised: + self.answer(agent, model) + self.assertIn(f"{model} is not one of the {agent} agent's models", str(raised.exception)) + self.assertIn(f"choose the agent {other} for it", str(raised.exception)) + with self.assertRaises(RuntimeError) as raised: + self.answer("muse_spark", "gpt-5") + self.assertTrue(str(raised.exception).endswith("choose one of them")) + self.assertEqual((self.vms, self.runs), ([], [])) + + def test_the_option_offers_every_agent_and_thinking_only_with_m3(self): + opts = bench()["options"] + self.assertEqual(opts["agent"]["values"], list(self.driver.AGENTS["agents"])) + self.assertEqual(opts["thinking"]["when"], {"agent": ["m3"]}) + models = [m for spec in self.driver.AGENTS["agents"].values() for m in spec["models"]] + self.assertEqual(models, bench()["setups"]["original"]["models"]) + + +class Definition(unittest.TestCase): + def test_muse_spark_is_sent_as_meta_serves_it_pinned_to_meta_and_not_compared_with_the_paper(self): + b = bench() + self.assertEqual(definitions.problems(b, "benchmark"), []) + other, why = definitions.setup(b, "original", MUSE, "openrouter") + self.assertEqual(why, "") + self.assertEqual(other["sent"], SENT) + self.assertEqual(other["pins"], {SENT: ["meta"], SONNET: ["anthropic"]}) + self.assertEqual(other["model"]["side_models"], [f"claude-sonnet-4-6={SONNET}"]) + self.assertFalse(other["comparable"]) + self.assertTrue(definitions.setup(b, "original", "MiniMax-M3", "openrouter")[0]["comparable"]) + self.assertEqual(run.pins_of(b, other, "openrouter", SONNET), {SENT: ["meta"], SONNET: ["anthropic"]}) + + def test_comparable_names_some_of_the_setups_models(self): + for value, ok in ((["MiniMax-M3"], True), (True, True), (False, True), ([MUSE], False), (["gpt-5"], False), ([], False), + (["MiniMax-M3", "MiniMax-M3"], False)): + b = bench() + b["setups"]["original"]["comparable"] = value + with self.subTest(value): + self.assertEqual(not [p for p in definitions.problems(b, "benchmark") if "comparable" in p], ok) + + def test_the_builder_offers_like_the_paper_with_m3_only_and_runs_muse_spark_as_any_model(self): + page, = recipe.page_of("osworld2")["setups"] + self.assertEqual((page["models"], page["sent"]["openrouter"]), (["MiniMax-M3"], {"MiniMax-M3": "minimax/minimax-m3"})) + r = {"version": 1, "harness": recipe.HARNESS, "name": "x", "title": "x", "benchmark": "osworld2", "datasets": ["test_v2"], + "others": ["original"], "provider": "openrouter", "limit": 1, "concurrency": 1, "unsandboxed": False, + "where": "local", "options": {}, "bench_options": {"agent": "muse_spark"}, "model": MUSE} + self.assertIn("Any of your choice", recipe.compile(dict(r, model_choice="paper"))["errors"][0]["message"]) + out = recipe.compile(dict(r, model_choice="any")) + self.assertEqual(out["errors"], []) + self.assertIn(f"run osworld2 original --model {MUSE} --provider openrouter", out["script"]) + self.assertIn("--bench-option agent=muse_spark", out["script"]) + picked = recipe.compile(dict(r, model_choice="any", model=SENT)) + self.assertEqual(picked["errors"], []) + self.assertIn(f"run osworld2 original --model {MUSE} --provider openrouter", picked["script"]) + + def test_the_relay_waits_longer_than_muse_spark(self): + b = bench() + self.assertEqual(b["limits"]["call_seconds"], 3300) + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + self.assertNotIn("--request_timeout", json.load(f)["agents"]["muse_spark"]["argv"]) + for bad in (0, -1, 1.5, True, "3300"): + b["limits"]["call_seconds"] = bad + with self.subTest(bad): + self.assertIn("limits.call_seconds: a whole number from 1", definitions.problems(b, "benchmark")) + + +class OpenRouter(BaseHTTPRequestHandler): + """A fake OpenRouter /responses; ``slow`` holds the reply.""" + seen = [] + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + OpenRouter.seen.append({"path": self.path, "body": body}) + time.sleep(float(body.get("slow") or 0)) + out = {"id": "gen-42", "object": "response", "status": "completed", "model": body["model"], + "output": [{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "e"}, + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "computer.stop", + "arguments": json.dumps({"answer": "done"})}], + "usage": {"input_tokens": 100, "input_tokens_details": {"cached_tokens": 90}, "output_tokens": 7, + "output_tokens_details": {"reasoning_tokens": 5}, "cost": 0.0002}} + if body.get("no_cost"): + del out["usage"]["cost"] + if self.headers.get("X-OpenRouter-Metadata") == "enabled": + out["openrouter_metadata"] = {"requested": body["model"], "endpoints": {"available": [ + {"provider": "Meta", "model": body["model"], "selected": True}]}} + data = json.dumps(out).encode() + try: + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + except OSError: + pass + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, json.loads(r.read()) + except urllib.error.HTTPError as e: + try: + return e.code, json.loads(e.read()) + finally: + e.close() + + +class Responses(unittest.TestCase): + """The unit's relay as an Original run of Muse Spark on OpenRouter starts it, on the local backend.""" + + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def unit(self, seconds, prices=None): + b = bench() + other, _ = definitions.setup(b, "original", MUSE, "openrouter") + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + ns = types.SimpleNamespace(base=self.base, key="sk-real", other=other, provider="openrouter", backend="local", + model=other["sent"], judge_model=SONNET, call_seconds=seconds, + serving=run.pins_of(b, other, "openrouter", SONNET), prices=prices, ledger_file=lambda: None) + rl = run.Run.unit_relay(ns, "u1", trial, {}) + self.addCleanup(rl.stop) + return rl + + def test_a_call_goes_to_responses_pinned_to_meta_and_its_cache_is_recorded(self): + OpenRouter.seen.clear() + rl = self.unit(3300) + self.assertEqual(rl.pool.timeout, 3300) + status, reply = post(rl.url("u1") + "/responses", {"model": MUSE, "input": [], "store": False, + "include": ["reasoning.encrypted_content"], + "reasoning": {"effort": "max", "summary": "auto"}}) + self.assertEqual((status, [i["type"] for i in reply["output"]]), (200, ["reasoning", "function_call"])) + sent = OpenRouter.seen[-1] + self.assertEqual(sent["path"], "/api/v1/responses") + self.assertEqual((sent["body"]["model"], sent["body"]["provider"]), (SENT, {"order": ["meta"], "allow_fallbacks": False})) + self.assertEqual(sent["body"]["reasoning"], {"effort": "max", "summary": "auto"}) + call, = rl.calls("u1") + self.assertEqual(call["request"]["routing"], {"order": ["meta"], "allow_fallbacks": False}) + self.assertEqual({k: call["reply"]["usage"][k] for k in ("input", "cached", "output", "reasoning")}, + {"input": 100, "cached": 90, "output": 7, "reasoning": 5}) + self.assertEqual(call["reply"]["tool_calls"][0]["name"], "computer.stop") + + def test_a_reply_with_no_cost_is_priced_from_the_price_list_the_run_has(self): + rl = self.unit(30, {SENT: {"input": 1.25, "cached": 0.15, "output": 4.25}}) + post(rl.url("u1") + "/responses", {"model": MUSE, "input": [], "no_cost": True}) + call = rl.calls("u1")[-1] + self.assertEqual(call["cost_from"], "estimate") + self.assertAlmostEqual(call["cost_usd"], (10 * 1.25 + 90 * 0.15 + 7 * 4.25) / 1e6) + + def test_a_reply_slower_than_the_relays_wait_is_a_502_and_one_within_it_goes_through(self): + status, reply = post(self.unit(0.3).url("u1") + "/responses", {"model": MUSE, "input": [], "slow": 1}) + self.assertEqual(status, 502) + self.assertIn("could not reach the endpoint", reply["error"]["message"]) + self.assertEqual(post(self.unit(5).url("u1") + "/responses", {"model": MUSE, "input": [], "slow": 1})[0], 200) + + def test_a_relay_container_is_given_the_benchmarks_wait(self): + argv = container.relay_argv("img", name="n", job="j", app="/app", record="/r", key_file="/k", base="b", model="m", + routes={}, timeout=3300) + self.assertEqual(argv[argv.index("--timeout") + 1], "3300") + self.assertNotIn("--timeout", container.relay_argv("img", name="n", job="j", app="/app", record="/r", key_file="/k", base="b", + model="m", routes={})) + + def test_a_run_reads_the_wait_from_the_benchmarks_limits(self): + b = definitions.get("benchmark", "osworld2") + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.call_seconds, 3300) + del b["limits"]["call_seconds"] + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.call_seconds, relay.TIMEOUT) + + +class Upstream(unittest.TestCase): + """What a unit of the Original keeps of its agent's calls: Muse Spark answers with tool calls and no text.""" + + ACTION = {"t0": 1, "t1": 2, "status": 200, "model": SENT, "request": {}, + "reply": {"text": "", "tool_calls": [{"name": "computer.computer", "arguments": '{"actions": []}'}], + "usage": {"input": 9, "output": 3, "cached": 0, "reasoning": 2}}} + + def drive(self, calls, stop=None, said="answer: RuntimeError: wrote no result.txt"): + record = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, record, ignore_errors=True) + if stop is not None: + with open(os.path.join(record, relay.STOP), "w") as f: + f.write(stop) + failing = types.SimpleNamespace(request=mock.Mock(side_effect=jsonl.Failed(said))) + ns = types.SimpleNamespace(other={"key": "original", "label": "OSWorld2 original"}, asked=MUSE, model=SENT, + name="j", bench={"label": "OSWorld2"}, turn_seconds=10, driver=lambda: failing, + attempt_kw=lambda: {}, simulator=lambda rl, tag: {}, judge=lambda rl, tag: {}) + rl = types.SimpleNamespace(calls=lambda tag: calls, url=lambda tag: "http://127.0.0.1:9/u1/v1", dir=record) + with self.assertRaises(run._Ended) as ended: + run.Run.drive_upstream(ns, "tasks__045", "/ws", "u1", [], rl) + return ended.exception + + def test_its_actions_are_the_trajectorys_tool_calls(self): + steps = job.trajectory_from_exchanges([], [self.ACTION], {"key": "original"}, SENT, "s")["steps"] + self.assertEqual(steps[-1]["tool_calls"], [{"tool_call_id": "call-2-1", "function_name": "computer.computer", + "arguments": {"actions": []}}]) + cut = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[{"name": "computer.computer", "arguments": '{"act'}])) + self.assertEqual(job.trajectory_from_exchanges([], [cut], {"key": "original"}, SENT, "s")["steps"][-1]["tool_calls"][0]["arguments"], + {"raw": '{"act'}) + bare = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[{"name": "computer.stop", "arguments": None}])) + self.assertEqual(job.trajectory_from_exchanges([], [bare], {"key": "original"}, SENT, "s")["steps"][-1]["tool_calls"][0]["arguments"], {}) + + def test_a_runner_that_fails_after_an_action_is_not_an_answer_the_model_ran_out_of(self): + ended = self.drive([self.ACTION]) + self.assertEqual(ended.kind, "NonZeroAgentExitCodeError") + self.assertIn("wrote no result.txt", ended.said) + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + self.assertEqual(self.drive([self.ACTION, silent]).kind, "OutputLengthExceededError") + self.assertEqual(self.drive([silent, self.ACTION]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(self.drive([self.ACTION, silent, dict(self.ACTION, side=True)]).kind, "OutputLengthExceededError") + self.assertEqual(self.drive([self.ACTION, dict(silent, side=True)]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(self.drive([self.ACTION, silent], stop=relay.OVERFLOW + "\n").kind, "OutputLengthExceededError") + + def test_the_drivers_setup_or_verifier_error_and_a_failed_last_reply_keep_their_kinds(self): + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + for kind in ("EnvironmentSetupError", "VerifierError"): + ended = self.drive([self.ACTION, silent], said=f"answer: {kind}: x") + self.assertEqual((ended.kind, ended.said), (kind, f"OSWorld2 original: answer: {kind}: x")) + failed = dict(silent, reply=dict(silent["reply"], end="failed", error={"code": "server_error"})) + ended = self.drive([self.ACTION, failed, dict(self.ACTION, side=True)]) + self.assertEqual(ended.kind, "ApiError") + self.assertIn("meta/muse-spark-1.3's last reply failed (server_error)", ended.said) + self.assertEqual(self.drive([failed, self.ACTION]).kind, "NonZeroAgentExitCodeError") + for late in (failed, silent): + self.assertEqual(self.drive([dict(self.ACTION, t0=3), dict(late, t0=2)]).kind, "NonZeroAgentExitCodeError") + self.assertEqual(run._kind(jsonl.Failed("score: EnvironmentSetupError: x"), "VerifierError"), "EnvironmentSetupError") + self.assertEqual(run._kind(jsonl.Failed("score: RuntimeError: EnvironmentSetupError: x"), "VerifierError"), + "VerifierError") + self.assertEqual(run._kind(jsonl.Failed("score: CancelledError: x"), "VerifierError"), "CancelledError") + self.assertEqual(run._kind(jsonl.Failed("answer: CauseUnknownError: x"), "NonZeroAgentExitCodeError"), "CauseUnknownError") + for kind in ("EnvironmentSetupError", "VerifierError"): + ended = self.drive([self.ACTION], stop="turn limit\n", said=f"answer: {kind}: x") + self.assertEqual((ended.kind, ended.said), (kind, f"OSWorld2 original: answer: {kind}: x")) + + def test_a_runner_that_fails_after_a_guards_stop_is_cancelled_not_out_of_output(self): + silent = dict(self.ACTION, reply=dict(self.ACTION["reply"], tool_calls=[])) + for stop, said in (("spent $2.39 >= $2.40\n", "spent $2.39 >= $2.40"), ("", relay.STOP)): + with self.subTest(said): + ended = self.drive([silent, self.ACTION, silent], stop=stop) + self.assertEqual(ended.kind, run.CANCELLED) + self.assertIn(f"a guard asked the unit to stop ({said})", ended.said) + self.assertIn("wrote no result.txt", ended.said) + self.assertTrue(ended.trajectory["steps"]) + + +def _runner(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + path = os.path.join(up, "scripts", "python", "run_multienv_muse_spark.py") + return path if up and os.path.isfile(path) else None + + +@unittest.skipUnless(_runner(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with run_multienv_muse_spark.py") +class Runner(unittest.TestCase): + def test_every_flag_passed_is_one_the_runners_parser_has(self): + with open(_runner(), encoding="utf-8") as f: + tree = ast.parse(f.read()) + known = {a.value for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "attr", "") == "add_argument" + for a in n.args if isinstance(a, ast.Constant) and str(a.value).startswith("--")} + with open(os.path.join(HERE, "agents.json"), encoding="utf-8") as f: + agents = json.load(f) + passed = {w for w in agents["common"] + agents["agents"]["muse_spark"]["argv"] if w.startswith("--")} + self.assertEqual(passed - known, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_nop.py b/app/tests/test_osworld2_nop.py new file mode 100644 index 0000000..567a765 --- /dev/null +++ b/app/tests/test_osworld2_nop.py @@ -0,0 +1,255 @@ +"""OSWorld2's null control: two setup attempts in nop.py, and a failed setup the score's error, never a 0. +A fake upstream and fake nop.py scripts stand in.""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +from nullius.standard import definitions, run + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +WORDS = "the task's own words" +FAILED = {"error": "EnvironmentSetupError", "cause": "TimeoutError", "attempts": 2} +TASK_LOADER = """def find_task_class_path(*a): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, **kw): + return {"instruction": "", "proxy": False} +""" +DESKTOP_ENV = """import json, os +MAX_RETRIES = 5 +class EnvironmentSetupError(RuntimeError): + pass +class DesktopEnv: + def __init__(self, **kw): + pass + def close(self): + out = os.environ["FAKE_OUT"] + with open(os.path.join(out, "closed"), "w") as f: + json.dump(os.path.exists(os.path.join(out, "setup_failed.json")), f) +""" +LIB_RUN_SINGLE = """import os +from desktop_env import desktop_env +def run_single_example(agent, env, example, max_steps, instruction, args, out, scores): + with open(os.path.join(out, "attempts"), "w") as f: + f.write(str(desktop_env.MAX_RETRIES)) + how = os.environ["FAKE_RUN"] + if how == "setup": + try: + raise TimeoutError(%(words)r) + except TimeoutError as e: + raise desktop_env.EnvironmentSetupError(%(words)r) from e + if how == "other": + raise ValueError(%(words)r) + with open(os.path.join(out, "result.txt"), "w") as f: + f.write("0.0") +""" % {"words": WORDS} +FAKE_NOP = """import json, os, sys, time +print("Started container with ports", flush=True) +print("[2026-11-07 01:02:03,456 ERROR desktopenv.setup] guest said: %(words)s", flush=True) +print("[2026-11-07 01:02:04,567 ERROR desktopenv.env] Environment setup failed with exception, retrying (1/2): " + "%(words)s", flush=True) +out = sys.argv[-1] +if os.environ.get("FAKE_FAILED"): + with open(os.path.join(out, "setup_failed.json"), "w") as f: + json.dump(%(failed)r, f) +with open(os.path.join(out, "result.txt"), "w") as f: + f.write("0.0") +time.sleep(float(os.environ.get("FAKE_HANG", "0"))) +sys.exit(int(os.environ.get("FAKE_EXIT", "0"))) +""" % {"failed": FAILED, "words": WORDS} + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_nop", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class ScoreLimit(unittest.TestCase): + def test_the_score_step_waits_an_hour(self): + b = definitions.get("benchmark", "osworld2") + self.assertEqual(b["limits"]["score_seconds"], 3600) + self.assertFalse([p for p in definitions.problems(b, "benchmark") if "limits" in p]) + r = run.Run(b, definitions.control(b, "nop"), model="", provider="openai", env={"OPENAI_API_KEY": "k"}, + which=lambda p: "/bin/" + p) + self.assertEqual(r.score_seconds, 3600) + + + def test_a_rate_limited_unit_is_not_run_again(self): + self.assertIs(definitions.get("benchmark", "osworld2")["limits"].get("run_again"), False) + + +class ScoringPrograms(unittest.TestCase): + def test_the_control_and_the_original_refuse_without_them(self): + b = definitions.get("benchmark", "osworld2") + for setup in ("yours", "original"): + with self.assertRaisesRegex(run.Refused, r"scoring_programs \(tesseract not on PATH\)"): + run.resolve_bench_options(b, "test_v2", {}, setup, lambda p: None if p == "tesseract" else "/bin/" + p) + self.assertIn("scoring_programs", run.resolve_bench_options(b, "test_v2", {}, setup, lambda p: "/bin/" + p)[1]["host"]) + self.assertEqual(b["host"]["scoring_programs"]["check"], ["identify", "convert", "ffmpeg", "ffprobe", "tesseract"]) + + +class TwoAttempts(unittest.TestCase): + """nop.py itself, on a fake upstream.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + up = os.path.join(self.tmp, "up") + os.makedirs(os.path.join(up, "desktop_env")) + for name, text in (("task_loader.py", TASK_LOADER), ("lib_run_single.py", LIB_RUN_SINGLE), + (os.path.join("desktop_env", "__init__.py"), ""), + (os.path.join("desktop_env", "desktop_env.py"), DESKTOP_ENV)): + with open(os.path.join(up, name), "w") as f: + f.write(text) + self.up = up + self.out = os.path.join(self.tmp, "out") + os.makedirs(self.out) + + def nop(self, how): + env = dict(os.environ, PYTHONPATH=self.up, FAKE_OUT=self.out, FAKE_RUN=how) + return subprocess.run([sys.executable, os.path.join(HERE, "nop.py"), "vm", "pw", "examples", "tasks", "001", + self.out], env=env, capture_output=True, text=True, timeout=60) + + def read(self, name): + with open(os.path.join(self.out, name)) as f: + return f.read() + + def test_a_failed_setup_is_said_before_the_close_with_no_words_of_the_task(self): + done = self.nop("setup") + self.assertEqual(done.returncode, 1) + self.assertEqual(json.loads(self.read("setup_failed.json")), FAILED) + self.assertNotIn(WORDS, self.read("setup_failed.json")) + self.assertEqual(self.read("attempts"), "2") + self.assertEqual(json.loads(self.read("closed")), True) + self.assertFalse(os.path.exists(os.path.join(self.out, "result.txt"))) + self.assertIn("the null control's setup of tasks__001 failed: EnvironmentSetupError", done.stderr) + + def test_another_failure_is_not_the_setups(self): + self.assertEqual(self.nop("other").returncode, 1) + self.assertFalse(os.path.exists(os.path.join(self.out, "setup_failed.json"))) + self.assertEqual(json.loads(self.read("closed")), False) + + def test_a_run_that_sets_up_scores(self): + self.assertEqual(self.nop("ok").returncode, 0) + self.assertEqual(self.read("attempts"), "2") + self.assertEqual(self.read("result.txt"), "0.0") + self.assertFalse(os.path.exists(os.path.join(self.out, "setup_failed.json"))) + + def test_the_driver_reads_the_file_nop_writes(self): + driver = load_driver(self.tmp) + spec = importlib.util.spec_from_file_location("osworld2_nop_constants", os.path.join(HERE, "nop.py")) + with open(spec.origin) as f: + self.assertIn(f'SETUP_FAILED = "{driver.SETUP_FAILED}"', f.read()) + + +class EndsEarly(unittest.TestCase): + """The driver's score of the null control, with a fake nop.py whose log quotes the task.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + self.driver = load_driver(self.tmp) + self.driver.START_LOCK = os.path.join(self.tmp, "start.lck") + self.driver.CLOSE_SECONDS = 1 + fake = os.path.join(self.tmp, "nop.py") + with open(fake, "w") as f: + f.write(FAKE_NOP) + self.reaped, started = [], [] + + def popen(*a, **kw): + started.append(subprocess.Popen(*a, **kw)) + return started[-1] + + def reaper(pgid, *_): + self.reaped.append(time.time()) + try: + os.killpg(pgid, signal.SIGKILL) + except OSError: + pass + + def end(): + for p in started: + if p.poll() is None: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + + self.addCleanup(end) + self.driver.subprocess = types.SimpleNamespace(**{k: getattr(subprocess, k) for k in dir(subprocess) + if not k.startswith("__") and k != "Popen"}, Popen=popen) + + for name, value in (("NOP", fake), ("_reaper", reaper), ("_remove_vms", lambda *a: None), + ("_noting", lambda *a: None), ("_vm", lambda *a: os.path.join(self.unit, "vm.qcow2")), + ("_task", lambda *_: {"instruction": "", "proxy": False}), ("_assets", lambda *a: "assets")): + p = mock.patch.object(self.driver, name, value) + p.start() + self.addCleanup(p.stop) + + def nop(self, **env): + with mock.patch.dict(os.environ, **env): + return self.driver.score("tasks__001", self.unit) + + def test_a_failed_setup_is_the_scores_error_not_a_0(self): + with self.assertRaises(RuntimeError) as raised: + self.nop(FAKE_FAILED="1") + said = str(raised.exception) + self.assertIn("the environment's setup failed (EnvironmentSetupError from TimeoutError, after 2 attempts), " + "and the task was not scored", said) + self.assertIsInstance(raised.exception, self.driver.EnvironmentSetupError) + self.assertNotIn(WORDS, said) + self.assertEqual(len(self.reaped), 1) + + def test_a_process_that_does_not_end_after_its_failure_is_handed_to_the_reaper(self): + t0 = time.time() + with self.assertRaises(RuntimeError) as raised: + self.nop(FAKE_FAILED="1", FAKE_HANG="30") + self.assertLess(time.time() - t0, 10) + self.assertIn("did not end 1 s after its failure, and was handed to the reaper", str(raised.exception)) + self.assertIn("EnvironmentSetupError", str(raised.exception)) + self.assertNotIn(WORDS, str(raised.exception)) + self.assertTrue(self.reaped) + + def test_a_failure_said_by_an_earlier_run_is_not_this_ones(self): + where = os.path.join(self.unit, "results", "nop", "tasks", "001") + os.makedirs(where) + with open(os.path.join(where, "setup_failed.json"), "w") as f: + json.dump(FAILED, f) + self.assertEqual(self.nop()["reward"], 0.0) + + def test_a_setup_that_succeeds_scores(self): + self.assertEqual(self.nop()["reward"], 0.0) + self.assertEqual(len(self.reaped), 1) + + def test_the_original_setup_is_never_ended_by_the_null_controls_file(self): + given = [] + + def run(argv, cwd, env, log, what, **kw): + given.append(kw) + raise RuntimeError("stop here") + + with mock.patch.object(self.driver, "_run", run), \ + mock.patch.object(self.driver, "_command", lambda *a: ({"script": "run.py", "env": {}}, os.path.join( + self.unit, "meta.json"), ["run.py"], {})), \ + self.assertRaises(RuntimeError): + self.driver.answer("tasks__001", self.unit, "original", "MiniMax-M3", "http://127.0.0.1:9/v1", "k") + self.assertEqual([{k: v for k, v in kw.items() if k != "until"} for kw in given], [{"stagger": True}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_proxy.py b/app/tests/test_osworld2_proxy.py new file mode 100644 index 0000000..2416b9e --- /dev/null +++ b/app/tests/test_osworld2_proxy.py @@ -0,0 +1,725 @@ +"""OSWorld2's proxy for the tasks that ask for one: tinyproxy in a container of the unit's own, removed however +the driver ends. A fake docker on PATH and a fake task_loader stand in.""" +from __future__ import annotations + +import contextlib +import importlib.util +import json +import logging +import os +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +DOCKER = """#!/bin/sh +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + image) [ -z "$FAKE_NO_IMAGE" ] || exit 1; printf '%s\n' "$FAKE_IMAGE" ;; + run) case "$*" in *"--entrypoint apk"*) echo "tinyproxy-1.11.3-r0 x86_64 {tinyproxy} (GPL-2.0-or-later) [installed]"; exit 0 ;; esac + [ -z "$FAKE_RUN_SLEEP" ] || exec sleep "$FAKE_RUN_SLEEP" + [ -z "$FAKE_RUN_FAIL" ] || { printf 'docker: Error: no such network\n\nRun %sdocker run --help%s for more information\n' "'" "'" >&2; exit 1; } + printf '%s' "$TINYPROXY_CONF" > "$FAKE_CONF"; echo cid ;; + inspect) [ -z "$FAKE_INSPECT_SLEEP" ] || sleep "$FAKE_INSPECT_SLEEP" + if [ -s "$FAKE_INSPECT_SEQ" ]; then head -n 1 "$FAKE_INSPECT_SEQ"; sed -i 1d "$FAKE_INSPECT_SEQ" + else printf '%s\\n' "$FAKE_INSPECT"; fi ;; + ps) case "$*" in *--format*) [ -z "$FAKE_NAMES" ] || printf '%s\\n' "$FAKE_NAMES" ;; + *) case "$FAKE_PS" in hang) exec sleep 5 ;; fail) echo "Error: daemon busy" >&2; exit 1 ;; esac + printf '%s\\n' $FAKE_IDS ;; esac ;; + rm) [ -z "$FAKE_RM_FAIL" ] || { echo "Error: busy" >&2; exit 1; } ;; + build) cat > "$FAKE_BUILD" ;; +esac +""" +LOADER = """import json, os +def find_task_class_path(eid, base, domain, version): + return True +def resolve_task_json_path(**kw): + return "" +def load_task_config(path, task_id, **kw): + return json.loads(os.environ["FAKE_TASK"]) +""" + + +def load(name, path, up=None): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up or HERE): + spec.loader.exec_module(module) + return module + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class Proxy(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.unit = os.path.join(self.tmp, "unit") + os.makedirs(self.unit) + self.calls = os.path.join(self.tmp, "calls") + self.conf = os.path.join(self.tmp, "conf") + self.listener = socket.socket() + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen() + self.addCleanup(self.listener.close) + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_CONF=self.conf, FAKE_INSPECT="running\t0\t127.0.0.1", FAKE_NAMES="", FAKE_IDS="", + FAKE_BUILD=os.path.join(self.tmp, "built"), + FAKE_INSPECT_SEQ=os.path.join(self.tmp, "inspect"), + FAKE_IMAGE='sha256:fake\t{"Labels":{"nullius.osworld2.proxy.package":"tinyproxy-1.11.3-r0"}}') + env.start() + self.addCleanup(env.stop) + for name in ("FAKE_NO_IMAGE", "FAKE_RUN_FAIL", "FAKE_RUN_SLEEP", "FAKE_RM_FAIL", "FAKE_PS", "FAKE_INSPECT_SLEEP", + "TINYPROXY_CONF"): + os.environ.pop(name, None) + self.proxy = load("osworld2_proxy", os.path.join(HERE, "proxy.py")) + self.proxy.PORT = self.listener.getsockname()[1] + self.proxy.START_SECONDS = 3 + self.later, self.watchers = [], [] + self.proxy._later = lambda dir, name=None: self.later.append(dir) + self.proxy._watcher = lambda dir, fd, name: self.watchers.append( + (dir, os.path.exists(self.pool()), [ln for ln in self.lines() if ln.startswith("run ")])) + + def lines(self): + try: + with open(self.calls) as f: + return f.read().splitlines() + except OSError: + return [] + + def ran(self): + return [ln for ln in self.lines() if ln.startswith("run ")][-1] + + def pool(self): + return os.path.join(self.unit, self.proxy.FILE) + + def held_open(self): + fd = os.open(self.pool(), os.O_RDONLY) + self.addCleanup(os.close, fd) + return fd + + def log(self): + try: + with open(os.path.join(self.unit, self.proxy.LOG)) as f: + return f.read().splitlines() + except OSError: + return [] + + def test_a_proxy_is_on_the_vms_network_unpublished_read_only_and_marked_with_its_unit(self): + path = self.proxy.start(self.unit) + argv = self.ran().split() + for flag in (["--network", "nullius-osworld2"], ["--read-only"], ["--cap-drop", "ALL"], ["--security-opt", "no-new-privileges"], + ["--log-driver", "local", "--log-opt", "max-size=1m", "--log-opt", "max-file=1", + "--log-opt", "compress=false"], + ["--label", f"{self.proxy.UNIT}={self.proxy._mark(self.unit)}"], + ["-e", self.proxy.CONF, self.proxy.IMAGE]): + self.assertIn(" ".join(flag), " ".join(argv)) + self.assertTrue(any(a.startswith("/tmp:") for a in argv)) + self.assertFalse({"-p", "--publish", "-P", "--publish-all", "--privileged", "-v", "--volume"} & set(argv)) + self.assertNotIn("--device-write-bps", argv) + self.assertTrue(argv[argv.index("--name") + 1].startswith(f"{self.proxy.NAME}{os.getuid()}-{os.getpid()}-")) + self.assertEqual(path, self.pool()) + + def test_the_network_is_the_vms_own_and_the_address_is_read_on_it(self): + site = load("osworld2_startup_proxy", os.path.join(HERE, "startup", "sitecustomize.py")) + self.assertEqual(self.proxy.NETWORK, site.NETWORK) + self.proxy.start(self.unit) + inspect = [ln for ln in self.lines() if ln.startswith("inspect ")][-1] + self.assertIn('{{with index .NetworkSettings.Networks "nullius-osworld2"}}{{.IPAddress}}{{end}}', inspect) + self.assertNotIn("bridge", inspect) + + def test_the_pool_file_names_the_container_and_the_units_password_and_only_its_owner_reads_it(self): + self.proxy.start(self.unit) + self.assertEqual(stat.S_IMODE(os.stat(self.pool()).st_mode), 0o600) + with open(self.pool()) as f: + entry, = json.load(f) + self.assertEqual({k: entry[k] for k in ("host", "port", "username", "protocol")}, + {"host": "127.0.0.1", "port": self.proxy.PORT, "username": self.proxy.USER, "protocol": "http"}) + self.assertRegex(entry["password"], r"^[0-9a-f]{32}$") + with open(self.conf) as f: + conf = f.read() + self.assertIn(f"BasicAuth {self.proxy.USER} {entry['password']}\n", conf) + self.assertNotIn(entry["password"], "\n".join(self.lines())) + self.proxy.remove(self.unit) + self.proxy.start(self.unit) + with open(self.pool()) as f: + self.assertNotEqual(json.load(f)[0]["password"], entry["password"]) + + def test_no_log_record_of_a_process_the_driver_starts_shows_the_units_password(self): + self.proxy.start(self.unit) + with open(self.pool()) as f: + word = json.load(f)[0]["password"] + out = os.path.join(self.unit, "file.log") + code = ("import logging, sys\n" + "a = logging.getLogger('desktopenv'); a.setLevel(logging.INFO)\n" + "a.addHandler(logging.StreamHandler(sys.stdout)); a.addHandler(logging.FileHandler(sys.argv[2]))\n" + "a.handlers[0].setFormatter(logging.Formatter('[%(levelname)s] %(message)s'))\n" + "a.info('Command executed successfully: %s', 'Upstream http nullius:' + sys.argv[1] + '@h:1')\n" + "a.info(f'export http_proxy=http://nullius:{sys.argv[1]}@h:1')\n" + "try:\n raise RuntimeError(sys.argv[1])\nexcept RuntimeError:\n a.exception('failed')\n") + env = dict(os.environ, PYTHONPATH=os.path.join(HERE, "startup"), PROXY_CONFIG_FILE=self.pool()) + said = subprocess.run([sys.executable, "-c", code, word, out], env=env, capture_output=True, text=True) + with open(out) as f: + written = said.stdout + said.stderr + f.read() + self.assertEqual(said.returncode, 0, said.stderr) + self.assertNotIn(word, written) + self.assertEqual(written.count(""), 6) + env.pop("PROXY_CONFIG_FILE") + said = subprocess.run([sys.executable, "-c", code, word, out], env=env, capture_output=True, text=True) + self.assertEqual(said.stdout.count(word), 3) + + def test_a_pool_file_without_a_password_hides_nothing_and_a_second_look_wraps_once(self): + site = load("osworld2_startup_hide", os.path.join(HERE, "startup", "sitecustomize.py")) + self.addCleanup(setattr, logging.Formatter, "format", logging.Formatter.format) + self.addCleanup(setattr, logging, "raiseExceptions", logging.raiseExceptions) + plain, path = logging.Formatter.format, os.path.join(self.unit, "pool.json") + for text in ("{", "{}", '[{"password": 1}]', '[{"password": ""}]', "[1]"): + with open(path, "w") as f: + f.write(text) + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + self.assertIs(logging.Formatter.format, plain) + site.hide_proxy_password({"PROXY_CONFIG_FILE": os.path.join(self.unit, "none.json")}) + site.hide_proxy_password({}) + self.assertIs(logging.Formatter.format, plain) + with open(path, "w") as f: + json.dump([{"password": "p" * 32}], f) + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + hidden = logging.Formatter.format + site.hide_proxy_password({"PROXY_CONFIG_FILE": path}) + self.assertIs(logging.Formatter.format, hidden) + self.assertFalse(logging.raiseExceptions) + record = logging.LogRecord("a", logging.INFO, "", 0, "x %s", ("p" * 32,), None) + self.assertEqual(logging.Formatter().format(record), "x ") + + def test_tinyproxy_logs_warnings_only_to_its_own_output_and_goes_out_directly(self): + lines = self.proxy.config("t" * 32).splitlines() + self.assertIn("LogLevel Warning", lines) + self.assertIn(f"Port {self.proxy.PORT}", lines) + self.assertFalse([ln for ln in lines if ln.split()[0] in ("Upstream", "LogFile", "Syslog", "Listen")]) + + def test_connect_goes_to_the_webs_ports_only_and_clients_with_the_main_thread_stay_below_its_processes(self): + lines = self.proxy.config("t" * 32).splitlines() + self.assertEqual(sorted(ln for ln in lines if ln.startswith("ConnectPort ")), ["ConnectPort 443", "ConnectPort 80"]) + self.assertIn(f"MaxClients {self.proxy.CLIENTS}", lines) + self.proxy.start(self.unit) + argv = self.ran().split() + self.assertEqual(int(argv[argv.index("--pids-limit") + 1]), self.proxy.PIDS) + self.assertLess(self.proxy.CLIENTS + 1, self.proxy.PIDS, + "a thread per client and tinyproxy's main thread all count against the container's pids limit") + + def test_the_image_is_alpine_pinned_by_digest_named_by_its_text_and_built_only_by_hand(self): + self.assertRegex(self.proxy.DOCKERFILE.splitlines()[0], r"^FROM alpine:[\d.]+@sha256:[0-9a-f]{64}$") + self.assertIn("tinyproxy -d -c /tmp/tinyproxy.conf", self.proxy.DOCKERFILE) + self.assertIn("USER 65534", self.proxy.DOCKERFILE) + os.environ["FAKE_NO_IMAGE"] = "1" + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("python3 proxy.py --build", str(raised.exception)) + self.assertFalse([ln for ln in self.lines() if ln.startswith(("run ", "build "))]) + self.assertFalse(os.path.exists(self.pool())) + built = subprocess.run([sys.executable, os.path.join(HERE, "proxy.py"), "--build"], capture_output=True, text=True) + self.assertEqual(built.returncode, 0, built.stderr) + builds = [ln for ln in self.lines() if ln.startswith("build ")] + self.assertEqual(builds, [f"build -t {self.proxy.IMAGE} -", + f"build --label {self.proxy.PACKAGE}=tinyproxy-1.11.3-r0 -t {self.proxy.IMAGE} -"]) + self.assertIn(f"run --rm --network none --entrypoint apk {self.proxy.IMAGE} list --installed tinyproxy", self.lines()) + self.assertEqual(built.stdout.split(), [self.proxy.IMAGE, "tinyproxy-1.11.3-r0"]) + with open(os.environ["FAKE_BUILD"]) as f: + self.assertEqual(f.read(), self.proxy.DOCKERFILE) + + def test_each_start_writes_the_images_id_and_tinyproxy_package_into_the_units_log(self): + self.proxy.start(self.unit) + self.assertRegex(self.log()[-1], r"\] proxy image sha256:fake, tinyproxy-1\.11\.3-r0$") + proxy = load("osworld2_proxy_unlabelled", os.path.join(HERE, "proxy.py")) + proxy.PORT, proxy._watcher = self.proxy.PORT, self.proxy._watcher + os.environ["FAKE_IMAGE"] = 'sha256:old\t{"User":"65534:65534"}' + proxy.start(self.unit) + self.assertIn("proxy image sha256:old, its tinyproxy package not labelled", self.log()[-1]) + + def test_a_process_looks_at_the_image_and_its_users_orphans_at_its_first_start_only(self): + for _ in range(2): + self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("image inspect")]), 1) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("ps ") and "--format" in ln]), 1) + self.assertEqual(len([ln for ln in self.log() if "proxy image" in ln]), 2) + + def test_the_proxy_and_its_units_vm_write_no_more_together_than_the_options_give_the_vm(self): + site = load("osworld2_startup_proxy", os.path.join(HERE, "startup", "sitecustomize.py")) + options = {"NULLIUS_OPTION_WRITE_MBPS": "8", "NULLIUS_OPTION_WRITE_IOPS": "80"} + with self.proxy.held(self.unit, True, lambda: "/dev/sda") as routed: + argv = self.ran().split() + vm = {key: rate for key, rate, _, _ in site.wanted(dict(options, **routed))} + own = {key: int(argv[argv.index(flag) + 1].rpartition(":")[2]) + for key, flag in (("device_write_bps", "--device-write-bps"), ("device_write_iops", "--device-write-iops"))} + self.assertEqual(own, {"device_write_bps": 52428, "device_write_iops": 1}) + self.assertEqual({k: vm[k] + own[k] for k in vm}, {"device_write_bps": 8 * 1024 * 1024, "device_write_iops": 80}) + self.assertEqual({key: rate for key, rate, _, _ in site.wanted(options)}, + {"device_write_bps": 8 * 1024 * 1024, "device_write_iops": 80}) + with self.proxy.held(self.unit, True, lambda: None) as routed: + self.assertEqual(routed, {"PROXY_CONFIG_FILE": self.pool()}) + self.assertNotIn("--device-write-bps", self.ran()) + for iops, taken in (("1", "1"), ("80", "0x")): + with self.subTest(iops=iops, taken=taken), self.assertRaises(site.WriteLimitError): + site.wanted(dict(options, NULLIUS_OPTION_WRITE_IOPS=iops, OSWORLD2_PROXY_WRITE_IOPS=taken)) + + def test_the_machines_limits_together_stay_within_its_budget_with_the_sites_and_a_proxy_per_unit(self): + site = load("osworld2_startup_budget", os.path.join(HERE, "startup", "sitecustomize.py")) + web = load("osworld2_websites_budget", os.path.join(HERE, "websites.py")) + mb = 1024 * 1024 + for (budget, units, vm, stack) in (((120, 1200), 7, (16, 160), (8, 80)), ((30, 300), 1, (22, 220), (8, 80)), + ((30, 300), 2, (11, 110), (8, 80))): + options = {"NULLIUS_OPTION_WRITE_MBPS": str(vm[0]), "NULLIUS_OPTION_WRITE_IOPS": str(vm[1])} + tiers = web.allocation(*stack) + sites = web.limits({f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() + for i, k in enumerate(("BPS", "IOPS"))})[1] + for routed in ({}, self.proxy.TAKEN): + with self.subTest(budget=budget, units=units, proxy=bool(routed)): + own = {key: rate for key, rate, _, _ in site.wanted(dict(options, **routed))} + each = (own["device_write_bps"] + (self.proxy.WRITE_BPS if routed else 0), + own["device_write_iops"] + (self.proxy.WRITE_IOPS if routed else 0)) + total = (units * each[0] + sites[0], units * each[1] + sites[1]) + self.assertLessEqual(total[0], budget[0] * mb) + self.assertLessEqual(total[1], budget[1]) + + def test_a_removal_that_fails_on_the_way_out_is_a_line_in_the_units_log(self): + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + with self.proxy.held(self.unit, True): + pass + lines = [ln for ln in self.log() if "proxy image" not in ln] + self.assertEqual(len(lines), 1) + self.assertRegex(lines[0], r"^\[\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\] the unit's proxy was not removed: .*busy") + self.assertTrue(os.path.exists(self.pool())) + + def test_a_proxy_that_stops_or_never_answers_is_an_error_and_is_removed(self): + os.environ["FAKE_INSPECT"] = "exited\t1\t" + os.environ["FAKE_IDS"] = "cid" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertIn("exit code 1", str(raised.exception)) + self.assertFalse([ln for ln in self.lines() if ln.startswith("logs ")]) + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + os.environ["FAKE_INSPECT"] = "running\t0\t127.0.0.1" + self.proxy.PORT = free_port() + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("did not answer", str(raised.exception)) + + def test_a_refused_container_is_an_error_with_dockers_words(self): + os.environ["FAKE_RUN_FAIL"] = "1" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertTrue(str(raised.exception).endswith("docker: Error: no such network"), str(raised.exception)) + self.assertFalse(os.path.exists(self.pool())) + + def test_held_gives_the_pool_while_the_body_runs_and_removes_the_proxy_however_it_ends(self): + os.environ["FAKE_IDS"] = "cid" + with self.proxy.held(self.unit, True, lambda: None) as routed: + self.assertEqual(routed, {"PROXY_CONFIG_FILE": self.pool()}) + self.assertTrue(os.path.exists(self.pool())) + self.assertFalse(os.path.exists(self.pool())) + with self.assertRaises(KeyError), self.proxy.held(self.unit, True): + raise KeyError("the run") + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual([ln for ln in self.lines() if ln.startswith("rm ")], ["rm -f cid", "rm -f cid"]) + + def test_a_task_without_a_proxy_costs_no_docker_call_nor_the_disk(self): + def disk(): + raise AssertionError("asked") + + with self.proxy.held(self.unit, False, disk) as routed: + self.assertEqual(routed, {}) + self.proxy.remove(self.unit) + self.assertEqual(self.lines(), []) + + def test_a_proxy_docker_did_not_remove_keeps_the_pool_file_for_the_next_sweep(self): + self.proxy.start(self.unit) + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + with self.assertRaises(RuntimeError): + self.proxy.remove(self.unit) + self.assertTrue(os.path.exists(self.pool())) + del os.environ["FAKE_RM_FAIL"] + self.proxy.remove(self.unit) + self.assertFalse(os.path.exists(self.pool())) + search = [ln for ln in self.lines() if ln.startswith("ps -a -q")][-1] + self.assertIn(f"label={self.proxy.UNIT}={self.proxy._mark(self.unit)}", search) + + def test_a_start_removes_this_users_proxies_whose_driver_is_gone_and_only_those(self): + ended = subprocess.Popen([sys.executable, "-c", ""]) + ended.wait() + prefix = f"{self.proxy.NAME}{os.getuid()}-" + dead, alive = f"{prefix}{ended.pid}-1", f"{prefix}{os.getpid()}-2" + other = f"{self.proxy.NAME}{os.getuid() + 1}-{ended.pid}-3" + os.environ["FAKE_NAMES"] = "\n".join([dead, alive, other, f"{prefix}junk"]) + self.proxy.start(self.unit) + self.assertIn(f"rm -f {dead}", self.lines()) + self.assertFalse([ln for ln in self.lines() if ln.startswith("rm ") and (alive in ln or other in ln)]) + + def test_a_container_the_daemon_creates_after_a_start_it_did_not_answer_is_still_removed(self): + self.proxy.DOCKER_SECONDS = 0.5 + os.environ["FAKE_RUN_SLEEP"] = "5" + with self.assertRaises(RuntimeError) as raised, self.proxy.held(self.unit, True): + pass + self.assertIn("docker run got no answer in 0.5 s", str(raised.exception)) + self.assertIn("has made no container for the proxy yet; the unit's sweeper removes one it makes later", + str(raised.exception)) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("run ")]), 1) + self.assertEqual(os.path.getsize(self.pool()), 0) + self.assertEqual(self.later, [self.unit]) + del os.environ["FAKE_RUN_SLEEP"] + self.proxy.remove(self.unit) + self.assertTrue(os.path.exists(self.pool())) + os.environ["FAKE_IDS"] = "late" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, self.held_open()) + self.assertIn("rm -f late", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def sweep_with_a_clock(self, docker_answers_at=None): + clock = [0.0] + + def sleep(seconds): + clock[0] += seconds + if docker_answers_at is not None and clock[0] >= docker_answers_at: + os.environ.pop("FAKE_RM_FAIL", None) + + self.proxy.time = types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0], strftime=time.strftime) + with open(self.pool(), "w") as f: + f.write("[]") + os.environ.update(FAKE_IDS="cid", FAKE_RM_FAIL="1") + self.proxy.sweep(self.unit, self.held_open(), "name") + return clock[0], [ln for ln in self.lines() if ln.startswith("rm ")] + + def test_a_sweeper_keeps_removing_while_docker_refuses_and_stops_once_it_succeeds(self): + ended, removals = self.sweep_with_a_clock(docker_answers_at=sum(self.proxy.SWEEPS) + 2 * self.proxy.RETRY_SECONDS) + self.assertFalse(os.path.exists(self.pool())) + self.assertFalse(os.path.exists(os.path.join(self.unit, self.proxy.LOG))) + self.assertEqual(ended, sum(self.proxy.SWEEPS) + 2 * self.proxy.RETRY_SECONDS) + self.assertEqual(len(removals), len(self.proxy.SWEEPS) + 2) + + def test_a_sweeper_whose_units_record_is_gone_waits_for_no_further_pass(self): + ended, removals = self.sweep_with_a_clock(docker_answers_at=0) + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual(ended, self.proxy.SWEEPS[0]) + self.assertEqual(len(removals), 1) + + def test_a_sweeper_docker_refuses_throughout_gives_up_in_time_with_a_line_in_the_units_log(self): + ended, removals = self.sweep_with_a_clock() + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS) + self.assertGreater(ended, self.proxy.SWEEP_SECONDS - self.proxy.RETRY_SECONDS) + self.assertGreater(len(removals), len(self.proxy.SWEEPS)) + self.assertTrue(os.path.exists(self.pool())) + lines = self.log() + self.assertEqual(len(lines), 1) + self.assertRegex(lines[0], rf"the unit's proxy was not removed in {self.proxy.SWEEP_SECONDS // 60} min: .*busy$") + + def test_a_start_docker_answers_late_goes_on_with_its_container_and_never_runs_a_second(self): + self.proxy.DOCKER_SECONDS = 0.5 + os.environ.update(FAKE_RUN_SLEEP="5", FAKE_IDS="cid") + path = self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("run ")]), 1) + name = self.ran().split()[self.ran().split().index("--name") + 1] + self.assertIn(f"ps -a -q --filter label={self.proxy.UNIT}={self.proxy._mark(self.unit)} --filter name={name}", + self.lines()) + with open(path) as f: + self.assertEqual(json.load(f)[0]["host"], "127.0.0.1") + + def sweep_with_slow_docker(self, found): + clock, calls = [0.0], [] + + def sleep(seconds): + clock[0] += seconds + + def run(argv, timeout=None, **kw): + calls.append((clock[0], timeout)) + if argv[1] == "ps" and timeout >= 119: + clock[0] += 119 + return subprocess.CompletedProcess(argv, 0, found, "") + clock[0] += timeout + raise subprocess.TimeoutExpired(argv, timeout) + + self.proxy.time = types.SimpleNamespace(sleep=sleep, monotonic=lambda: clock[0], strftime=time.strftime) + self.proxy.subprocess = types.SimpleNamespace(run=run, TimeoutExpired=subprocess.TimeoutExpired, + CompletedProcess=subprocess.CompletedProcess) + open(self.pool(), "w").close() + self.proxy.sweep(self.unit, self.held_open(), "first") + for began, timeout in calls: + self.assertLessEqual(timeout, self.proxy.DOCKER_SECONDS) + self.assertLessEqual(began + timeout, self.proxy.SWEEP_SECONDS + 0.1) + return clock[0], calls + + def test_a_sweep_whose_searches_take_minutes_and_removals_never_answer_ends_in_its_time(self): + ended, calls = self.sweep_with_slow_docker("cid\n") + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS + 0.1) + self.assertGreater(len(calls), 2 * len(self.proxy.SWEEPS) - 2) + self.assertTrue(os.path.exists(self.pool())) + self.assertRegex(self.log()[-1], rf"was not removed in {self.proxy.SWEEP_SECONDS // 60} min: docker ") + + def test_a_sweep_whose_searches_take_minutes_still_removes_its_record_in_its_time(self): + ended, calls = self.sweep_with_slow_docker("") + self.assertLessEqual(ended, self.proxy.SWEEP_SECONDS) + self.assertFalse(os.path.exists(self.pool())) + self.assertEqual(self.log(), []) + + def test_a_sweeper_leaves_a_later_start_in_the_same_folder_its_proxy_and_its_record(self): + with open(self.pool(), "w") as f: + f.write("[]") + own = self.held_open() + os.remove(self.pool()) + with open(self.pool(), "w") as f: + f.write('[{"host": "later"}]') + os.environ["FAKE_IDS"] = "later" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, own, "first") + self.assertFalse([ln for ln in self.lines() if ln.startswith(("rm ", "ps -a -q"))]) + self.assertTrue(os.path.exists(self.pool())) + with self.proxy._locked(self.unit): + self.assertFalse(self.proxy._ours(self.unit, own)) + + def test_a_sweeper_removes_its_own_starts_container_by_its_name(self): + with open(self.pool(), "w") as f: + f.write("[]") + os.environ["FAKE_IDS"] = "cid" + self.proxy.SWEEPS = (0, 0) + self.proxy.sweep(self.unit, self.held_open(), "first") + self.assertIn(f"ps -a -q --filter label={self.proxy.UNIT}={self.proxy._mark(self.unit)} --filter name=first", + self.lines()) + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def test_a_start_whose_run_and_search_docker_both_leave_unanswered_is_an_error_for_its_sweeper(self): + self.proxy.DOCKER_SECONDS = 0.5 + for said, words in (("hang", "docker ps got no answer in 0.5 s: is its daemon running?: whether it made the " + "proxy's container is unknown"), + ("fail", "could not say whether it made the proxy's container: Error: daemon busy")): + with self.subTest(said=said): + os.environ.update(FAKE_RUN_SLEEP="5", FAKE_PS=said) + with contextlib.suppress(FileNotFoundError): + os.remove(self.pool()) + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertIn("docker run got no answer in 0.5 s", str(raised.exception)) + self.assertIn(words, str(raised.exception)) + self.assertTrue(str(raised.exception).endswith("the unit's sweeper removes one it makes later")) + self.assertEqual(os.path.getsize(self.pool()), 0) + + def test_a_start_with_slow_answers_ends_within_its_time_however_docker_answers(self): + self.proxy.START_SECONDS = 1.5 + os.environ["FAKE_INSPECT_SLEEP"] = "0.4" + os.environ["FAKE_INSPECT"] = "created\t0\t" + t0 = time.monotonic() + with self.assertRaises(RuntimeError) as raised: + self.proxy.start(self.unit) + self.assertLess(time.monotonic() - t0, self.proxy.START_SECONDS + 0.5) + self.assertRegex(str(raised.exception), "did not answer in its start's 1.5 s|the time given for it ran out|" + "got no answer in") + + def test_a_container_created_and_not_yet_started_is_waited_for(self): + with open(os.environ["FAKE_INSPECT_SEQ"], "w") as f: + f.write("created\t0\t\ncreated\t0\t\n") + self.proxy.start(self.unit) + self.assertEqual(len([ln for ln in self.lines() if ln.startswith("inspect ")]), 3) + + def test_the_waits_are_sized_for_a_shared_daemon_that_takes_minutes_to_answer(self): + proxy = load("osworld2_proxy_waits", os.path.join(HERE, "proxy.py")) + self.assertGreaterEqual(proxy.DOCKER_SECONDS, 120) + self.assertGreater(proxy.START_SECONDS, 2 * proxy.DOCKER_SECONDS) + self.assertGreaterEqual(proxy.RETRY_SECONDS, proxy.DOCKER_SECONDS) + self.assertGreaterEqual(sum(proxy.SWEEPS), 900) + self.assertGreater(proxy.SWEEP_SECONDS, sum(proxy.SWEEPS) + proxy.RETRY_SECONDS) + + def test_a_late_container_whose_unit_holds_no_record_is_an_orphan_while_its_driver_lives(self): + prefix = f"{self.proxy.NAME}{os.getuid()}-{os.getpid()}" + kept, late = f"{prefix}-1", f"{prefix}-2" + os.makedirs(os.path.join(self.tmp, "other")) + open(os.path.join(self.tmp, "other", self.proxy.FILE), "w").close() + os.environ["FAKE_NAMES"] = f"{kept}\t{os.path.join(self.tmp, 'other')}\n{late}\t{self.unit}" + self.proxy._orphans() + self.assertIn(f"rm -f {late}", self.lines()) + self.assertFalse([ln for ln in self.lines() if ln.startswith("rm ") and kept in ln]) + + def test_a_proxy_that_started_and_was_removed_needs_no_sweeper(self): + with self.proxy.held(self.unit, True): + pass + self.assertEqual(self.later, []) + self.assertIn(f"--label {self.proxy.FOLDER}={self.unit}", self.ran()) + + def test_the_sweeper_is_a_session_of_its_own_holding_its_starts_record_and_naming_its_container(self): + proxy = load("osworld2_proxy_later", os.path.join(HERE, "proxy.py")) + open(self.pool(), "w").close() + with mock.patch.object(proxy.subprocess, "Popen") as popen: + proxy._later(self.unit, "first") + argv, kw = popen.call_args + fd, = kw["pass_fds"] + self.assertEqual(argv[0][1:], [os.path.join(HERE, "proxy.py"), "--sweep", self.unit, str(fd), "first"]) + self.assertTrue(kw["start_new_session"]) + + def test_a_start_arms_its_watcher_once_its_record_exists_and_before_docker_is_asked(self): + self.proxy.start(self.unit) + self.assertEqual(self.watchers, [(self.unit, True, [])]) + + def test_the_watcher_sweeps_the_units_proxy_once_its_driver_is_gone_and_only_then(self): + ended = subprocess.Popen([sys.executable, "-c", ""]) + ended.wait() + self.proxy.SWEEPS = (0, 0) + open(self.pool(), "w").close() + gone = self.held_open() + os.remove(self.pool()) + self.proxy.watch(self.unit, ended.pid, gone, "first") + self.assertEqual(self.lines(), []) + open(self.pool(), "w").close() + os.environ["FAKE_IDS"] = "cid" + self.proxy.watch(self.unit, ended.pid, self.held_open(), "first") + self.assertIn("rm -f cid", self.lines()) + self.assertFalse(os.path.exists(self.pool())) + + def test_a_driver_killed_while_its_proxy_starts_leaves_the_proxy_to_its_watcher(self): + os.environ.update(FAKE_RUN_SLEEP="60", FAKE_IDS="cid") + driver = subprocess.Popen([sys.executable, "-c", "import sys; sys.path.insert(0, sys.argv[1]); import proxy; " + "proxy.start(sys.argv[2])", HERE, self.unit], start_new_session=True) + self.addCleanup(self.end_watchers) + t0 = time.time() + while not any(ln.startswith("run ") for ln in self.lines()) and time.time() - t0 < 20: + time.sleep(0.1) + self.assertTrue(os.path.exists(self.pool())) + os.killpg(driver.pid, signal.SIGKILL) + while "rm -f cid" not in self.lines() and time.time() - t0 < 40: + time.sleep(0.2) + driver.wait() + self.assertIn("rm -f cid", self.lines()) + + def end_watchers(self): + for pid in filter(str.isdigit, os.listdir("/proc")): + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + argv = f.read().split(b"\0") + except OSError: + continue + if b"--watch" in argv and self.unit.encode() in argv: + with contextlib.suppress(OSError): + os.kill(int(pid), signal.SIGKILL) + + def test_an_attempts_leftover_proxy_is_removed_before_a_new_one(self): + open(self.pool(), "w").close() + os.environ["FAKE_IDS"] = "old" + self.proxy.start(self.unit) + lines = self.lines() + self.assertLess(lines.index("rm -f old"), lines.index(self.ran())) + + +class Driver(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load("osworld2_driver_proxy", os.path.join(HERE, "driver.py"), self.tmp) + self.envs, self.held, self.asked = [], [], [] + + def start(dir, disk=None): + self.held.append(("start", disk)) + return os.path.join(dir, "proxy.json") + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + + for target, name, fake in ((self.driver, "_vm", lambda dir, d, e: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_disk", lambda: self.asked.append(1) or "/dev/sda"), + (self.driver.proxy, "start", start), + (self.driver.proxy, "remove", lambda dir: self.held.append(("remove", dir)))): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + + def unit(self, asks): + p = mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": asks}) + p.start() + self.addCleanup(p.stop) + + def both(self): + with self.assertRaises(RuntimeError): + self.driver.answer("tasks__037", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + with self.assertRaises(RuntimeError): + self.driver.score("tasks__037", self.tmp) + + def test_a_task_that_asks_for_a_proxy_runs_with_its_own_in_the_original_and_the_null_control(self): + self.unit(True) + self.both() + self.assertEqual([e.get("PROXY_CONFIG_FILE") for e in self.envs], [os.path.join(self.tmp, "proxy.json")] * 2) + self.assertEqual(self.held, [("start", "/dev/sda"), ("remove", self.tmp)] * 2) + + def test_a_task_that_asks_for_none_gets_no_proxy_nor_the_variable(self): + self.unit(False) + with mock.patch.dict(os.environ): + os.environ.pop("PROXY_CONFIG_FILE", None) + self.both() + self.assertEqual([e.get("PROXY_CONFIG_FILE") for e in self.envs], [None, None]) + self.assertEqual((self.held, self.asked), ([], [])) + + def test_the_vms_removal_removes_the_units_proxy(self): + with mock.patch.object(self.driver, "_mine", lambda dir: ([], [])): + self.driver._remove_vms(self.tmp) + self.assertEqual(self.held, [("remove", self.tmp)]) + + def test_a_write_limit_that_leaves_the_vm_nothing_beside_the_proxy_is_refused_before_it_starts(self): + driver = load("osworld2_driver_share", os.path.join(HERE, "driver.py"), self.tmp) + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="8", NULLIUS_OPTION_WRITE_IOPS="1"), \ + mock.patch.object(driver, "_asked", mock.Mock(side_effect=AssertionError("asked"))): + with self.assertRaises(driver._startup().WriteLimitError) as raised: + driver._disk() + self.assertIn("OSWORLD2_PROXY_WRITE_IOPS", str(raised.exception)) + + def test_the_proxys_disk_is_the_vms_one_only_with_a_write_limit_set(self): + driver = load("osworld2_driver_disk", os.path.join(HERE, "driver.py"), self.tmp) + site = mock.Mock() + site.root_disk.return_value = "/dev/sdb" + asked = mock.Mock(return_value=subprocess.CompletedProcess([], 0, "/var/lib/docker\n", "")) + with mock.patch.object(driver, "_startup", lambda: site), mock.patch.object(driver, "_asked", asked): + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="none", NULLIUS_OPTION_WRITE_IOPS="none"): + self.assertIsNone(driver._disk()) + asked.assert_not_called() + with mock.patch.dict(os.environ, NULLIUS_OPTION_WRITE_MBPS="30", NULLIUS_OPTION_WRITE_IOPS="none"): + self.assertEqual(driver._disk(), "/dev/sdb") + site.root_disk.assert_called_once_with("/var/lib/docker") + + +class Flag(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + with open(os.path.join(self.tmp, "task_loader.py"), "w") as f: + f.write(LOADER) + + def loaded(self, task): + out = os.path.join(self.tmp, "out.json") + done = subprocess.run([sys.executable, os.path.join(HERE, "nop.py"), "--instruction", self.tmp, "tasks", "037", out], + env=dict(os.environ, PYTHONPATH=self.tmp, FAKE_TASK=json.dumps(task)), + capture_output=True, text=True) + self.assertEqual(done.returncode, 0, done.stderr) + with open(out) as f: + return json.load(f) + + def test_nop_says_whether_the_task_asks_for_a_proxy_as_the_upstream_reads_it(self): + self.assertEqual(self.loaded({"instruction": "x", "proxy": True}), {"instruction": "x", "proxy": True}) + self.assertEqual(self.loaded({"instruction": "x"}), {"instruction": "x", "proxy": False}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_stop.py b/app/tests/test_osworld2_stop.py new file mode 100644 index 0000000..9c98467 --- /dev/null +++ b/app/tests/test_osworld2_stop.py @@ -0,0 +1,420 @@ +"""OSWorld2's episode ends with DONE and is scored once the unit's relay holds its stop file: the agent's own call +refused for its context window, or a spend guard that asks. A fake agent and episode stand in for the upstream's, +a fake provider on loopback for the endpoint: nothing starts a VM or calls a model.""" +from __future__ import annotations + +import contextlib +import fcntl +import importlib.util +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius.standard import relay + +HERE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +EPISODE = """def run_single_example(agent, env, max_steps): + taken, done, step = [], False, 0 + while not done and step < max_steps: + response, actions = agent.predict("", {}) + if not actions: + raise RuntimeError("the simulated user was asked nothing") + for action in actions: + taken.append(action) + if action in ("DONE", "FAIL"): + done = True + break + step += 1 + env.append(taken) +""" +RUNNER = """import json, os, sys +import lib_run_single +class Agent: + def predict(self, instruction, obs): + open(os.environ["OSWORLD2_STOP"], "a").close() + return "", [] +env = [] +lib_run_single.run_single_example(Agent(), env, 5) +print(json.dumps(env)) +""" + + +def load(name, path, **env): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", [path]), mock.patch.object(sys, "path", [os.path.dirname(path), *sys.path]), \ + mock.patch.dict(os.environ, env): + spec.loader.exec_module(module) + return module + + +class Agent: + def __init__(self, *steps): + self.steps, self.calls = list(steps), 0 + + def predict(self, instruction, obs): + self.calls += 1 + step = self.steps.pop(0) + return step() if callable(step) else step + + +class Episode(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.stop = os.path.join(self.tmp, "stop") + self.site = load("osworld2_startup_stop", os.path.join(STARTUP, "sitecustomize.py")) + self.module = types.ModuleType("lib_run_single") + exec(EPISODE, self.module.__dict__) + with mock.patch.dict(os.environ, OSWORLD2_STOP=self.stop): + self.site.stop_when_asked(self.module) + + def asked(self, *_): + open(self.stop, "a").close() + + def ran(self, agent, steps=5): + env = [] + self.module.run_single_example(agent, env, steps) + return env[0] + + def test_with_no_stop_the_agents_steps_run_as_they_are(self): + agent = Agent(*[("", ["click"])] * 3) + self.assertEqual(self.ran(agent, 3), ["click"] * 3) + self.assertEqual(agent.calls, 3) + + def test_an_overflow_during_the_call_ends_the_episode_with_done(self): + agent = Agent(("", ["click"]), lambda: (self.asked(), ("", []))[1], ("", ["click"])) + self.assertEqual(self.ran(agent), ["click", "DONE"]) + self.assertEqual(agent.calls, 2) + self.assertTrue(os.path.exists(self.stop + ".taken")) + + def test_the_action_already_bought_is_taken_before_done(self): + agent = Agent(lambda: (self.asked(), ("", ["type"]))[1]) + self.assertEqual(self.ran(agent), ["type", "DONE"]) + os.remove(self.stop) + agent = Agent(lambda: (self.asked(), ("", ["FAIL"]))[1]) + self.assertEqual(self.ran(agent), ["FAIL"]) + os.remove(self.stop) + agent = Agent(lambda: (self.asked(), ({"text": ""}, [{"action_type": "CLICK"}]))[1]) + self.assertEqual(self.ran(agent), [{"action_type": "CLICK"}, "DONE"]) + + def test_a_call_that_raised_after_the_stop_ends_with_done(self): + def raised(): + self.asked() + raise RuntimeError("the endpoint failed too many times") + self.assertEqual(self.ran(Agent(raised)), ["DONE"]) + self.assertTrue(os.path.exists(self.stop + ".taken")) + + def test_a_call_that_raised_with_no_stop_is_the_upstreams(self): + def raised(): + raise RuntimeError("boom") + with self.assertRaises(RuntimeError): + self.ran(Agent(raised)) + + def test_only_a_stop_that_ended_a_step_is_recorded_with_its_reasons(self): + taken = self.stop + ".taken" + self.assertEqual(self.ran(Agent(("", ["DONE"]))), ["DONE"]) + with open(self.stop, "w") as f: + f.write("batch cap: spent\n") + self.assertFalse(os.path.exists(taken)) + self.assertEqual(self.ran(Agent(("", ["click"]))), ["DONE"]) + with open(self.stop, "a") as f: + f.write("turn limit\n") + self.assertEqual(self.ran(Agent(("", ["click"]))), ["DONE"]) + with open(taken) as f: + self.assertEqual(f.read(), "batch cap: spent\n") + + def test_a_stop_before_the_step_asks_the_model_nothing(self): + self.asked() + agent = Agent(("", ["click"])) + self.assertEqual(self.ran(agent), ["DONE"]) + self.assertEqual(agent.calls, 0) + + def test_wrapped_once_and_not_at_all_without_a_stop_path(self): + wrapped = self.module.run_single_example + with mock.patch.dict(os.environ, OSWORLD2_STOP=self.stop): + self.site.stop_when_asked(self.module) + self.assertIs(self.module.run_single_example, wrapped) + agent = Agent(("", ["click"]), ("", ["click"])) + self.ran(agent, 1) + predict = agent.predict + self.ran(agent, 1) + self.assertIs(agent.predict, predict) + bare = types.ModuleType("lib_run_single") + exec(EPISODE, bare.__dict__) + run = bare.run_single_example + with mock.patch.dict(os.environ, {"OSWORLD2_STOP": ""}): + self.site.stop_when_asked(bare) + self.assertIs(bare.run_single_example, run) + + +class Imported(unittest.TestCase): + def test_the_runners_import_is_wrapped_as_the_driver_starts_it(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + for name, text in (("lib_run_single.py", EPISODE), ("runner.py", RUNNER)): + with open(os.path.join(tmp, name), "w") as f: + f.write(text) + env = dict(os.environ, PYTHONPATH=os.pathsep.join([STARTUP, tmp]), OSWORLD2_STOP=os.path.join(tmp, "stop"), + NULLIUS_OPTION_VM_PROVIDER="docker") + out = subprocess.run([sys.executable, os.path.join(tmp, "runner.py")], env=env, capture_output=True, text=True, + timeout=60) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertEqual(json.loads(out.stdout), [["DONE"]]) + + +class Driver(unittest.TestCase): + def setUp(self): + trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, trial, ignore_errors=True) + self.dir = os.path.join(trial, "workspace") + self.stop = os.path.join(trial, ".work", "relay", "stop") + os.makedirs(os.path.dirname(self.stop)) + os.makedirs(self.dir) + self.driver = load("osworld2_driver_stop", os.path.join(HERE, "driver.py"), NULLIUS_UPSTREAM=self.dir) + + def test_the_runner_is_told_the_units_stop_file(self): + envs = [] + + def started(argv, cwd, env, *a, **kw): + envs.append(env) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", started), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}), \ + mock.patch.object(self.driver, "_assets", lambda *a: "assets"), self.assertRaises(KeyboardInterrupt): + self.driver.answer("tasks__001", self.dir, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + self.assertEqual(envs[0]["OSWORLD2_STOP"], self.stop) + + def test_a_stopped_episodes_score_says_why_and_keeps_its_result(self): + where = os.path.join(self.dir, "results", "pyautogui", "screenshot", "MiniMax-M3", "tasks", "001") + os.makedirs(where) + with open(os.path.join(where, "result.txt"), "w") as f: + f.write("0.5") + scored = self.driver.score("tasks__001", self.dir, setup="original") + self.assertEqual(scored["reward"], 0.5) + self.assertNotIn("stopped", scored) + with open(self.stop, "w") as f: + f.write("turn limit\n") + self.assertNotIn("stopped", self.driver.score("tasks__001", self.dir, setup="original")) + with open(self.stop + ".taken", "w") as f: + f.write("context window\ncontext window\nbatch cap: spent $1\n") + relayed = os.path.dirname(self.stop) + for name, calls in (("u0001", []), ("u0001", [{"status": 200, "side": True}]), ("u0001-judge", [{"status": 200}])): + with open(os.path.join(relayed, name + ".jsonl"), "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls)) + with self.subTest(name=name, calls=calls), self.assertRaises(self.driver.CancelledError) as raised: + self.driver.score("tasks__001", self.dir, setup="original") + self.assertIn("stopped (context window) before the agent's first step", str(raised.exception)) + with open(os.path.join(relayed, "u0001.jsonl"), "a") as f: + f.write(json.dumps({"status": 200}) + '\n{"stat\n') + scored = self.driver.score("tasks__001", self.dir, setup="original") + self.assertEqual((scored["reward"], scored["stopped"], scored["stopped_by"]), + (0.5, "context window", ["context window", "money"])) + with open(self.stop + ".taken", "w") as f: + f.write("forecast $2 >= $1\nturn limit\nsomething else\n") + self.assertEqual(self.driver.score("tasks__001", self.dir, setup="original")["stopped_by"], + ["money", "time", "other"]) + open(self.stop + ".taken", "w").close() + with open(self.stop, "w") as f: + f.write("turn limit\n") + self.assertEqual(self.driver.score("tasks__001", self.dir, setup="original")["stopped_by"], ["time"]) + with open(os.path.join(relayed, "u0001.jsonl"), "w") as f: + f.write(json.dumps({"status": 200, "side": True, "said": "a\u2028b"}, ensure_ascii=False) + "\n") + with open(self.stop + ".taken", "w") as f: + f.write("turn limit\n") + with self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver.score("tasks__001", self.dir, setup="original") + self.assertIn("stopped (turn limit) before the agent's first step", str(raised.exception)) + self.assertEqual(len(self.driver._traj(relayed, "u0001.jsonl")), 1) + + def test_the_episode_is_stopped_at_its_own_limit_counted_from_the_runners_start(self): + argv = [sys.executable, "-c", "import os, sys, time\nt = time.time()\n" + "while not os.path.exists(sys.argv[1]) and time.time() - t < 20: time.sleep(0.05)", self.stop] + log = os.path.join(self.dir, "run.log") + with mock.patch.object(self.driver, "_reaper", lambda *a, **kw: None): + for episode, until in ((1, time.time() + 600), (600, time.time() + 1), (1, None)): + if until is None: + shutil.rmtree(os.path.dirname(self.stop)) + until = time.time() + 600 + with self.subTest(episode=episode), mock.patch.object(self.driver, "EPISODE_SECONDS", episode): + self.driver._run(argv, self.dir, dict(os.environ), log, "the runner", until=until) + with open(self.stop) as f: + self.assertEqual(f.read(), "turn limit\n") + os.remove(self.stop) + + def test_the_answers_limit_leaves_the_turn_time_to_evaluate(self): + kws = [] + + def started(*a, **kw): + kws.append(kw) + raise KeyboardInterrupt + + with mock.patch.object(self.driver, "_vm", lambda *a: "vm"), mock.patch.object(self.driver, "_run", started), \ + mock.patch.object(self.driver, "_task", lambda *_: {"instruction": "", "proxy": False}), \ + mock.patch.object(self.driver, "_assets", lambda *a: "assets"), self.assertRaises(KeyboardInterrupt): + self.driver.answer("tasks__001", self.dir, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + left = kws[0]["until"] - time.time() + self.assertAlmostEqual(left, self.driver._turn_seconds() - self.driver.EVALUATE_SECONDS - self.driver.CLOSE_SECONDS, + delta=5) + self.assertGreater(left, self.driver.EPISODE_SECONDS) + + def test_a_runner_alive_an_evaluations_time_after_its_turn_limit_is_reaped_unscored(self): + argv = [sys.executable, "-c", "import time; time.sleep(15)"] + log, reaped = os.path.join(self.dir, "run.log"), [] + + def reaper(pgid, *a, **kw): + reaped.append(pgid) + with contextlib.suppress(OSError): + os.killpg(pgid, signal.SIGKILL) + + with mock.patch.object(self.driver, "_reaper", reaper): + for calls, kind in (([{"status": 200, "side": True}], self.driver.EnvironmentSetupError), + ([{"status": 200}], self.driver.VerifierError)): + with open(os.path.join(os.path.dirname(self.stop), "u0001.jsonl"), "w") as f: + f.write("".join(json.dumps(c) + "\n" for c in calls)) + t0 = time.time() + with self.subTest(calls=calls), self.assertRaises(kind) as raised: + self.driver._run(argv, self.dir, dict(os.environ), log, "the runner", until=time.time() + 0.5, grace=1) + self.assertLess(time.time() - t0, 10) + self.assertIn("the runner did not end 1 s after the unit's turn limit", str(raised.exception)) + self.assertTrue(reaped and self.driver._gone(reaped[-1], 5)) + with open(os.path.join(os.path.dirname(self.stop), "u0001.jsonl"), "w") as f: + f.write(json.dumps({"status": 200, "side": True}) + "\n") + done = "[2026-11-10 20:00:00,000 INFO desktopenv.env/1-EnvProcess-1] Environment setup complete." + for said, kind in (("", self.driver.EnvironmentSetupError), (done, self.driver.VerifierError)): + with self.subTest(said=said), self.assertRaises(kind): + self.driver._run([sys.executable, "-c", f"import time; print({said!r}, flush=True); time.sleep(15)"], self.dir, + dict(os.environ), log, "the null control's run", failed=os.path.join(self.dir, "failed"), + until=time.time() + 0.5, grace=1) + + def test_a_reaper_that_does_not_start_is_a_setup_error_after_the_start_lock_is_let_go(self): + argv = [sys.executable, "-c", "import time; time.sleep(15)"] + lock, held, groups = os.path.join(self.dir, "start.lck"), [], [] + for failing in (None, RuntimeError("no thread")): + def reap(pgid, *a, **kw): + groups.append(pgid) + with open(lock) as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + held.append(False) + except BlockingIOError: + held.append(True) + if failing: + raise failing + os.killpg(pgid, signal.SIGKILL) + + with self.subTest(failing=failing), mock.patch.object(self.driver, "START_LOCK", lock), \ + mock.patch.object(self.driver, "_remove_vms") as removed, \ + mock.patch.object(self.driver, "_arm", mock.Mock(side_effect=ValueError("no pipe"))), \ + mock.patch.object(self.driver, "reap", reap), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(argv, self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", stagger=True) + self.assertIn("the runner's reaper did not start: ValueError: no pipe", str(raised.exception)) + self.assertTrue(self.driver._gone(groups[-1], 5)) + self.assertEqual(removed.call_count, 1 if failing else 0) + self.assertEqual(held, [False, False]) + + def test_a_start_lock_that_is_no_plain_file_is_a_setup_error(self): + lock = os.path.join(self.dir, "start.lck") + os.mkdir(lock) + with mock.patch.object(self.driver, "START_LOCK", lock), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(["true"], self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", stagger=True) + self.assertIn("OSError: [Errno 22] not a plain file", str(raised.exception)) + + def test_a_start_lock_held_past_the_units_turn_limit_is_a_setup_error(self): + lock = os.path.join(self.dir, "start.lck") + fd = os.open(lock, os.O_RDONLY | os.O_CREAT, 0o644) + self.addCleanup(os.close, fd) + fcntl.flock(fd, fcntl.LOCK_EX) + closing = threading.Timer(10, self.driver.CLOSED.set) + closing.start() + self.addCleanup(closing.cancel) + with mock.patch.object(self.driver, "START_LOCK", lock), self.assertRaises(self.driver.EnvironmentSetupError) as raised: + self.driver._run(["true"], self.dir, dict(os.environ), os.path.join(self.dir, "run.log"), "the runner", + stagger=True, until=time.time() + 1) + self.assertIn("the runner was not started: the unit's time ran out", str(raised.exception)) + self.assertFalse(self.driver.CLOSED.is_set()) + + +class Provider(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + said = {"long": b'{"error":{"message":"This endpoint\'s maximum context length is 10 tokens"}}', + "image": b'{"error":{"message":"invalid image"}}'}.get(body["messages"][0]["content"]) + self.send_response(400 if said else 200) + said = said or json.dumps({"choices": [{"message": {"content": "ok"}}], "usage": {}}).encode() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(said))) + self.end_headers() + self.wfile.write(said) + + +class Relayed(unittest.TestCase): + def setUp(self): + self.up = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + threading.Thread(target=self.up.serve_forever, daemon=True).start() + self.tmp = tempfile.mkdtemp() + self.relay = relay.Relay(f"http://127.0.0.1:{self.up.server_address[1]}/v1", "sk", self.tmp, model="m", + side_models=["sim"]).start() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(self.up.shutdown) + self.addCleanup(self.relay.stop) + self.stop = os.path.join(self.tmp, relay.STOP) + + def post(self, url, model, text): + req = urllib.request.Request(url + "/chat/completions", headers={"Content-Type": "application/json"}, + data=json.dumps({"model": model, "messages": [{"role": "user", "content": text}]}).encode()) + try: + with urllib.request.urlopen(req) as r: + return r.status + except urllib.error.HTTPError as e: + e.close() + return e.code + + def test_only_the_agents_own_overflow_leaves_the_stop_file(self): + self.assertEqual(self.post(self.relay.url("u1"), "m", "image"), 400) + self.assertEqual(self.post(self.relay.url("u1"), "sim", "long"), 400) + self.assertEqual(self.post(self.relay.url("u1-judge", model="j"), "j", "long"), 400) + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 200) + self.assertFalse(os.path.exists(self.stop)) + self.assertEqual(self.post(self.relay.url("u1"), "m", "long"), 400) + with open(self.stop) as f: + self.assertEqual(f.readline(), "context window\n") + self.assertEqual(len(self.relay.calls("u1")), 4) + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 200) + + def test_a_stop_asked_after_an_overflow_is_heeded(self): + with open(self.stop, "w") as f: + f.write("context window\nbatch cap\n") + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 400) + self.assertEqual(self.relay.calls("u1"), []) + + def test_once_asked_from_outside_the_agents_own_calls_are_not_sent(self): + with open(self.stop, "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(self.relay.url("u1"), "m", "fine"), 400) + self.assertEqual(self.relay.calls("u1"), []) + self.assertEqual(self.post(self.relay.url("u1"), "sim", "fine"), 200) + self.assertEqual(self.post(self.relay.url("u1-judge", model="j"), "j", "fine"), 200) + self.assertEqual(len(self.relay.calls("u1")) + len(self.relay.calls("u1-judge")), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_websites.py b/app/tests/test_osworld2_websites.py new file mode 100644 index 0000000..3b550fd --- /dev/null +++ b/app/tests/test_osworld2_websites.py @@ -0,0 +1,821 @@ +"""OSWorld2's mock websites, as websites.py and the driver reach them, before a fake proxy and a fake docker. +With NULLIUS_OSWORLD2_UPSTREAM or NULLIUS_OSWORLD2_WEB set, also checked against them, read as syntax trees.""" +from __future__ import annotations + +import ast +import contextlib +import glob +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock +from urllib.parse import parse_qs, urlsplit + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +GATEWAY = "10.213.0.1" +SUFFIX = f"{GATEWAY}.nip.io" +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + ps) [ -n "$FAKE_ROW" ] && printf '%s\\n' "$FAKE_ROW" ;; + compose) [ -n "$FAKE_UP" ] && touch "$FAKE_UP" ;; +esac +exit 0 +""" + + +def load(): + spec = importlib.util.spec_from_file_location("osworld2_websites", os.path.join(HERE, "websites.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Sites(BaseHTTPRequestHandler): + """Each site by the host asked for: ``down`` 502, ``empty`` an empty page, ``stateless`` no state.""" + down, empty, stateless, overleaf_up = set(), set(), set(), None + + def log_message(self, *_): + pass + + def do_GET(self): + url = urlsplit(self.path) + name = url.hostname[:-len(SUFFIX) - 1] + if name in self.down or (name == "overleaf" and url.path == "/login" and self.overleaf_up + and not os.path.exists(self.overleaf_up)): + self.send_response(502) + self.end_headers() + return + if url.path == "/api/state": + cookie = parse_qs(url.query).get("cookie", [""])[0] + body = b"{}" if name in self.stateless else json.dumps({"user_id": cookie, "state": {"data": {}}}).encode() + else: + body = b"" if name in self.empty else b"site" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class Case(unittest.TestCase): + def setUp(self): + self.w = load() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + Sites.down, Sites.empty, Sites.stateless, Sites.overleaf_up = set(), set(), set(), None + server = ThreadingHTTPServer(("127.0.0.1", 0), Sites) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + self.unknown, self.elsewhere = set(), set() + + def resolves(host): + name = host[:-len(SUFFIX) - 1] + if not host.endswith(SUFFIX) or name in self.unknown: + return set() + return {"172.17.0.1"} if name in self.elsewhere else {GATEWAY} + + for patch in (mock.patch.object(self.w, "PROXIES", {"http": f"http://127.0.0.1:{server.server_port}"}), + mock.patch.object(self.w, "_resolves", resolves), + mock.patch.object(self.w, "POLL", 0.2), + mock.patch.object(self.w, "LOCK", os.path.join(self.tmp, "overleaf.lck")), + mock.patch.object(self.w, "SITES_LOCK", os.path.join(self.tmp, "websites.lck"))): + patch.start() + self.addCleanup(patch.stop) + + +class Option(Case): + def test_none_gives_no_suffix_and_a_task_that_needs_the_sites_is_refused(self): + self.assertEqual(self.w.env({}), {}) + self.assertIsNone(self.w.needed("001", {})) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.needed("005", {self.w.OPTION: "none"}) + self.assertIn("task 005 needs the mock websites", str(raised.exception)) + + def test_a_suffix_is_every_process_s_website_host_suffix(self): + given = {self.w.OPTION: SUFFIX} + self.assertEqual(self.w.env(given), {"WEBSITE_HOST_SUFFIX": SUFFIX}) + self.assertEqual(self.w.needed("057", given), SUFFIX) + self.assertIsNone(self.w.needed("001", given)) + + def test_a_value_that_is_no_host_suffix_is_refused(self): + for bad in ("http://x.nip.io", "x", "a b.io", "x.nip.io/", "-x.io", "X.IO"): + with self.subTest(bad), self.assertRaises(self.w.WebsiteError): + self.w.env({self.w.OPTION: bad}) + + def test_the_tasks(self): + self.assertEqual(len(self.w.TASKS), 40) + self.assertLess(self.w.OVERLEAF_TASKS, self.w.TASKS) + self.assertEqual(len(self.w.NAMES), 25) + self.assertEqual(len(set(self.w.NAMES)), 25) + + +class Ready(Case): + def test_every_name_and_each_stateful_site_answers(self): + self.assertEqual(self.w.ready(SUFFIX, overleaf=True), 25) + + def test_a_unit_s_check_leaves_overleaf_to_its_own_tasks(self): + Sites.down.add("overleaf") + self.assertEqual(self.w.ready(SUFFIX), 24) + with self.assertRaises(self.w.WebsiteError): + self.w.ready(SUFFIX, overleaf=True) + + def test_a_name_that_does_not_resolve(self): + self.unknown.add("trippza") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"trippza.{SUFFIX} does not resolve", str(raised.exception)) + + def test_a_suffix_whose_names_resolve_past_the_gateway_is_refused(self): + self.elsewhere.add("awsconsole") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"awsconsole.{SUFFIX} resolves to 172.17.0.1, not {GATEWAY}", str(raised.exception)) + self.assertIn(f"such as {GATEWAY}.nip.io", str(raised.exception)) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.suffix({self.w.OPTION: "x"}) + self.assertIn(f"such as {GATEWAY}.nip.io", str(raised.exception)) + + def test_a_site_that_fails(self): + Sites.down.add("mailhub") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"http://mailhub.{SUFFIX}/ answered 502", str(raised.exception)) + self.assertIn("websites.py compose STACK up -d", str(raised.exception)) + + def test_an_empty_page_is_a_name_the_stack_does_not_serve(self): + Sites.empty.add("interfacinglinux") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn("serves another suffix", str(raised.exception)) + + def test_a_site_with_no_state(self): + Sites.stateless.add("wandb") + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.ready(SUFFIX) + self.assertIn(f"wandb.{SUFFIX}/api/state?cookie={self.w.COOKIE} answered no state", str(raised.exception)) + + +class Overleaf(Case): + def setUp(self): + super().setUp() + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.calls = os.path.join(self.tmp, "calls") + self.settings = os.path.join(self.tmp, "OSWorld-web.osworld2.env") + with open(self.settings, "w") as f: + f.write(f"HOST_SUFFIX={SUFFIX}\nCADDY_SCHEME=http://\n") + self.up = os.path.join(self.tmp, "up") + Sites.overleaf_up = self.up + env = mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_UP=self.up, FAKE_ROW=f"/s/docker-compose.yml,/s.osworld2.compose.yml\t/s\t{self.settings}") + env.start() + self.addCleanup(env.stop) + + def docker(self): + with open(self.calls) as f: + return f.read().splitlines() + + def test_any_other_task_holds_nothing(self): + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertFalse(os.path.exists(self.calls)) + + def test_overleaf_starts_anew_as_the_project_runs(self): + with self.w.held("057", SUFFIX, threading.Event()): + calls = self.docker() + compose = f"compose -p osworld2-web --project-directory /s -f /s/docker-compose.yml -f /s.osworld2.compose.yml --env-file {self.settings}" + self.assertEqual(calls[1:], [ + f"{compose} rm --stop --force -v " + " ".join(self.w.OVERLEAF), + "volume rm --force osworld2-web_overleaf_web_data osworld2-web_overleaf_web_mongo_data osworld2-web_overleaf_web_redis_data", + f"{compose} up -d --no-build --pull never overleaf_web_app"]) + self.assertIn("label=com.docker.compose.project=osworld2-web", calls[0]) + self.assertNotIn("service=", calls[0]) + + def test_a_stack_for_another_suffix_is_not_touched(self): + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", "10.0.0.5.nip.io", threading.Event()): + pass + self.assertIn(f"the stack serves {SUFFIX}", str(raised.exception)) + self.assertEqual(len(self.docker()), 1) + + def test_no_overleaf_running(self): + with mock.patch.dict(os.environ, FAKE_ROW=""), self.assertRaises(self.w.WebsiteError) as raised: + self.w.reset_overleaf(SUFFIX) + self.assertIn("no container of the osworld2-web project", str(raised.exception)) + + def test_overleaf_is_waited_on_until_its_login_page_answers(self): + with mock.patch.object(self.w, "OVERLEAF_SECONDS", 1), mock.patch.dict(os.environ, FAKE_UP=""): + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.reset_overleaf(SUFFIX) + self.assertIn("Overleaf did not come up in 1 s", str(raised.exception)) + + def test_the_next_overleaf_task_waits_for_the_lock(self): + order, released = [], threading.Event() + + def first(): + with self.w.held("057", SUFFIX, threading.Event()): + order.append("057 in") + released.wait(10) + order.append("057 out") + + t = threading.Thread(target=first) + t.start() + while not order: + time.sleep(0.05) + threading.Timer(1.0, released.set).start() + with self.w.held("072", SUFFIX, threading.Event()): + order.append("072 in") + t.join(10) + self.assertEqual(order, ["057 in", "057 out", "072 in"]) + + def test_a_process_given_the_locks_keeps_them_after_the_unit_lets_go(self): + with self.w.held("057", SUFFIX, threading.Event()): + fds = self.w.held_fds() + self.assertEqual(len(fds), 2) + child = subprocess.Popen([sys.executable, "-c", "import sys; sys.stdin.read()"], stdin=subprocess.PIPE, + pass_fds=fds) + self.addCleanup(child.wait, 10) + self.assertEqual(self.w.held_fds(), ()) + with self.assertRaises(self.w.WebsiteError): + with self.w._alone(): + pass + closed = threading.Event() + threading.Timer(1.0, closed.set).start() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", SUFFIX, closed): + pass + self.assertIn("another unit's Overleaf task ran", str(raised.exception)) + child.stdin.close() + child.wait(10) + with self.w.held("072", SUFFIX, threading.Event()): + pass + + def test_reset_and_smoke_refuse_while_a_unit_uses_the_sites(self): + stack = os.path.join(self.tmp, "OSWorld-web") + with self.w.held("005", SUFFIX, threading.Event()), mock.patch.object(self.w, "_pinned", os.path.abspath): + for start in (self.w.reset, self.w.smoke): + with self.assertRaises(self.w.WebsiteError) as raised: + start(stack) + self.assertIn("a unit is using the mock websites", str(raised.exception)) + self.assertFalse(os.path.exists(self.calls)) + + def test_compose_by_hand_changes_nothing_while_a_unit_uses_the_sites(self): + ran = [] + with mock.patch.object(self.w, "compose", lambda stack, args, **kw: ran.append(args) or 0): + with self.w.held("005", SUFFIX, threading.Event()), mock.patch("sys.stderr", new=io.StringIO()) as err: + for args in (["down", "--volumes"], ["up", "-d", "--force-recreate"], ["-p", "ps", "rm"], []): + self.assertEqual(self.w.main(["compose", self.tmp, *args]), 1, args) + for args in (["ps"], ["--", "logs", "caddy"], ["stats", "--no-stream"]): + self.assertEqual(self.w.main(["compose", self.tmp, *args]), 0, args) + self.assertIn("a unit is using the mock websites", err.getvalue()) + self.assertEqual(self.w.main(["compose", self.tmp, "down"]), 0) + self.assertEqual(ran, [["ps"], ["logs", "caddy"], ["stats", "--no-stream"], ["down"]]) + + def test_the_locks_are_the_machine_s_whoever_runs(self): + names = [] + for uid in (1000, 1001): + with mock.patch.object(os, "getuid", lambda: uid): + w = load() + names.append((w.LOCK, w.SITES_LOCK)) + self.assertEqual(names[0], names[1]) + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertEqual(os.stat(self.w.SITES_LOCK).st_mode & 0o777, 0o644) + os.chmod(self.w.SITES_LOCK, 0o444) + other = subprocess.Popen([sys.executable, "-c", "import fcntl, os, sys; fd = os.open(sys.argv[1], os.O_RDONLY); " + "fcntl.flock(fd, fcntl.LOCK_SH); print(flush=True); sys.stdin.read()", + self.w.SITES_LOCK], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(other.wait, 10) + self.addCleanup(other.stdout.close) + self.addCleanup(other.kill) + other.stdout.readline() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w._alone(): + pass + self.assertIn("a unit is using the mock websites", str(raised.exception)) + with self.w.held("005", SUFFIX, threading.Event()): + pass + other.stdin.close() + other.wait(10) + with self.w._alone(): + pass + os.remove(self.w.SITES_LOCK) + os.symlink(os.path.join(self.tmp, "elsewhere"), self.w.SITES_LOCK) + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("005", SUFFIX, threading.Event()): + pass + self.assertIn("cannot be opened", str(raised.exception)) + self.assertFalse(os.path.exists(os.path.join(self.tmp, "elsewhere"))) + + def test_one_overleaf_task_at_a_time_and_no_longer_than_its_unit(self): + inside, waited = threading.Event(), threading.Event() + released = threading.Event() + + def first(): + with self.w.held("057", SUFFIX, threading.Event()): + inside.set() + released.wait(10) + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + closed = threading.Event() + threading.Timer(1.5, closed.set).start() + t0 = time.time() + with self.assertRaises(self.w.WebsiteError) as raised: + with self.w.held("072", SUFFIX, closed): + waited.set() + self.assertFalse(waited.is_set()) + self.assertIn("time ran out", str(raised.exception)) + self.assertLess(time.time() - t0, 5) + released.set() + t.join(10) + with self.w.held("072", SUFFIX, threading.Event()): + waited.set() + self.assertTrue(waited.is_set()) + + +class Stack(Case): + def test_a_folder_not_at_the_release_is_refused(self): + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.configure(self.tmp, SUFFIX, 2, 20) + self.assertIn(self.w.COMMIT, str(raised.exception)) + + def test_the_stack_listens_on_the_vms_network_s_gateway_alone(self): + stack, made = os.path.join(self.tmp, "OSWorld-web"), [] + with mock.patch.object(self.w, "_pinned", lambda stack: stack), \ + mock.patch.object(self.w, "_root_disk", lambda: ("/var/lib/docker", "/dev/sda")), \ + mock.patch.object(self.w, "_network", lambda: made.append(1) or "nullius-osworld2"): + with self.assertRaises(self.w.WebsiteError): + self.w.configure(stack, SUFFIX, 0, 80) + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.configure(stack, SUFFIX, 8, 40) + self.assertIn("at least 80 IOPS", str(raised.exception)) + self.assertEqual(made, []) + path, settings = self.w.configure(stack, SUFFIX, 8, 80) + self.assertEqual((settings["OSWORLD2_WEB_ADDRESS"], made), (GATEWAY, [1])) + with open(path) as f: + self.assertIn(f"OSWORLD2_WEB_ADDRESS={GATEWAY}\n", f.read()) + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + self.w.main(["configure", stack, SUFFIX, "--address", "172.17.0.1", "--write-mbps", "8", "--write-iops", "80"]) + + def test_the_sum_of_the_services_limits_never_passes_the_stack_s_share(self): + for mbps, iops in ((8, 80), (10, 100), (9, 81), (30, 300), (1, 80)): + tiers = self.w.allocation(mbps, iops) + settings = {f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))} + each, total = self.w.limits(settings) + self.assertEqual(sorted(each), sorted(self.w.SERVICES)) + self.assertLessEqual(total, (mbps * self.w.MB, iops)) + self.assertTrue(all(b > 0 and i > 0 for b, i in each.values())) + writers = [each[s] for s in self.w.WEIGHTS] + self.assertTrue(all(w >= each["awsconsole"] for w in writers)) + if (mbps, iops) == (8, 80): + self.assertEqual(total[1], 80) + self.assertGreater(total[0], 8 * self.w.MB - 1024) + + def test_the_groups_build_every_service_once(self): + self.assertEqual(len(self.w.SERVICES), 40) + self.assertEqual(len(set(self.w.SERVICES)), 40) + + def test_every_service_of_the_override_is_limited_and_caddy_pinned_on_one_address(self): + with open(os.path.join(HERE, "websites.compose.yml"), encoding="utf-8") as f: + text = f.read() + services = re.findall(r"^ ([a-z0-9_]+):$", text.split("\nservices:\n", 1)[1], re.M) + self.assertEqual(sorted(services), sorted(self.w.SERVICES)) + self.assertEqual(text.count(" blkio_config: *blkio\n logging: *logs\n"), 40 - len(self.w.WEIGHTS)) + self.assertEqual(text.count(" logging: *logs\n"), 40) + self.assertEqual(set(re.findall(r"\$\{(OSWORLD2_WEB_\w+|HOST_SUFFIX|CADDY_SCHEME)", text)) - {"HOST_SUFFIX", "CADDY_SCHEME"}, + set(self.w.SETTINGS) - {"HOST_SUFFIX", "CADDY_SCHEME"}) + for service in self.w.WEIGHTS: + block = re.split(r"\n (?=\S)", text.split(f"\n {service}:\n", 1)[1], maxsplit=1)[0] + self.assertIn(f"OSWORLD2_WEB_{service.upper()}_BPS", block) + self.assertRegex(self.w.CADDY.search(text).group(1), r"@sha256:[0-9a-f]{64}$") + self.assertIn(" ports: !override\n - ${OSWORLD2_WEB_ADDRESS:", text) + self.assertNotIn("443", text) + + +BUILD_DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1 $2" in + "network inspect") echo "1|10.213.0.0/24 10.213.0.1" ;; + "inspect -f") printf '%s\\n' "$FAKE_HOSTCONFIG" ;; + "buildx inspect") exit "${FAKE_NO_BUILDER:-0}" ;; + "image inspect") echo "sha256:$(echo "$5" | md5sum | cut -c1-12)" ;; + "compose version") echo "${FAKE_COMPOSE:-2.40.3+ds1-0ubuntu1}" ;; +esac +case "$*" in *" config --format json") printf '%s\\n' "$FAKE_CONFIG" ;; esac +exit 0 +""" + + +class Build(Case): + """The stack's own BuildKit with a write limit, and the build group by group, on a fake docker.""" + + def setUp(self): + super().setUp() + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(BUILD_DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.stack = os.path.join(self.tmp, "OSWorld-web") + os.makedirs(self.stack) + with open(self.stack + ".osworld2.env", "w") as f: + f.write(f"HOST_SUFFIX={SUFFIX}\n") + self.calls = os.path.join(self.tmp, "calls") + limits = {"BlkioDeviceWriteBps": [{"Path": "/dev/dm-0", "Rate": 20 * 1024 * 1024}], + "BlkioDeviceWriteIOps": [{"Path": "/dev/dm-0", "Rate": 200}]} + config = {"services": {"caddy": {"image": "lucaslorentz/caddy-docker-proxy@sha256:" + "c" * 64}, + "overleaf_web_mongo": {"image": "mongo:8.0.20"}}} + self.free = [100.0] + for patch in (mock.patch.dict(os.environ, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_HOSTCONFIG=json.dumps(limits), FAKE_CONFIG=json.dumps(config)), + mock.patch.object(self.w, "_pinned", os.path.abspath), + mock.patch.object(self.w, "_root_disk", lambda: ("/var/lib/docker", "/dev/dm-0")), + mock.patch.object(self.w, "_free_gb", lambda root: self.free.pop(0) if len(self.free) > 1 else self.free[0])): + patch.start() + self.addCleanup(patch.stop) + + def docker(self): + with open(self.calls) as f: + return f.read().splitlines() + + def test_the_builder_writes_under_its_limit(self): + self.w.builder(self.stack, "up", 20, 200) + run, inspect, create = self.docker() + self.assertIn("--privileged", run) + self.assertIn("--device-write-bps /dev/dm-0:20971520 --device-write-iops /dev/dm-0:200", run) + self.assertTrue(run.endswith(self.w.BUILDKIT)) + self.assertEqual(create, f"buildx create --name {self.w.BUILDER} --driver remote docker-container://{self.w.BUILDKITD}") + with open(self.stack + ".osworld2.buildkitd.toml") as f: + self.assertIn("max-parallelism = 4", f.read()) + + def test_a_builder_docker_kept_no_limit_for_is_removed_and_never_used(self): + with mock.patch.dict(os.environ, FAKE_HOSTCONFIG="{}"), self.assertRaises(self.w.WebsiteError) as raised: + self.w.builder(self.stack, "up", 20, 200) + self.assertIn("no write limit", str(raised.exception)) + self.assertEqual(self.docker()[-1], f"rm -f {self.w.BUILDKITD}") + self.assertFalse([c for c in self.docker() if c.startswith("buildx create")]) + + def test_the_groups_one_after_another_each_cache_removed(self): + said = [] + out = self.w.build(self.stack, say=said.append) + calls = self.docker() + steps = [c for c in calls if " pull " in c or " build " in c or c.startswith("buildx prune")] + self.assertEqual(len(steps), 1 + 2 * (len(self.w.GROUPS) - 1)) + self.assertTrue(steps[0].endswith("pull " + " ".join(self.w.GROUPS[0][1]))) + for (name, services, _), (built, pruned) in zip(self.w.GROUPS[1:], zip(steps[1::2], steps[2::2])): + self.assertTrue(built.endswith("build " + " ".join(services)), name) + self.assertEqual(pruned, f"buildx prune --builder {self.w.BUILDER} --all --force") + self.assertEqual([json.loads(s)["group"] for s in said], [g[0] for g in self.w.GROUPS]) + with open(out["images"]) as f: + images = json.load(f)["images"] + self.assertEqual(len(images), 40) + self.assertEqual(images["overleaf_web_mongo"]["image"], "mongo:8.0.20") + self.assertEqual(images["wandb"]["image"], "osworld2-web-wandb") + + def test_the_build_stops_before_a_group_that_would_eat_the_reserve(self): + self.free = [40.0, 30.0] + with self.assertRaises(self.w.WebsiteError) as raised: + self.w.build(self.stack, say=lambda line: None) + self.assertIn("stopped before overleaf: 30.0 GB free", str(raised.exception)) + self.assertIn("done: pulled", str(raised.exception)) + self.assertFalse([c for c in self.docker() if " build " in c]) + + def test_a_compose_without_override_tags_is_refused(self): + for old in ("2.20.3", "2.24.3", "v2.24.0"): + with self.subTest(old), mock.patch.dict(os.environ, FAKE_COMPOSE=old), \ + self.assertRaises(self.w.WebsiteError) as raised: + self.w.compose(self.stack, ["config"], capture=True) + self.assertIn("Docker Compose 2.24.4 or later", str(raised.exception)) + with mock.patch.dict(os.environ, FAKE_COMPOSE="2.24.4"): + self.assertEqual(self.w.compose(self.stack, ["config"], capture=True).returncode, 0) + + def test_compose_hands_the_locks_it_holds_to_docker(self): + given = [] + with mock.patch.object(self.w, "_compose_version", lambda: None), mock.patch.object(self.w, "_network", lambda: None), \ + mock.patch.object(self.w.subprocess, "run", lambda *a, **kw: given.append(kw["pass_fds"]) or mock.Mock()): + with self.w._alone(): + fds = self.w.held_fds() + self.w.compose(self.stack, ["down"]) + self.w.compose(self.stack, ["up", "-d"], capture=True) + self.assertEqual(len(fds), 1) + self.assertEqual(given, [fds, fds]) + + def test_every_compose_makes_the_vms_network_first(self): + self.w.compose(self.stack, ["up", "-d"], capture=True) + calls = self.docker() + made = next(i for i, c in enumerate(calls) if c.startswith("network inspect ") and c.endswith(" nullius-osworld2")) + self.assertLess(made, next(i for i, c in enumerate(calls) if c.endswith(" up -d"))) + + def test_the_names_are_tried_from_the_vms_network(self): + self.w._from_vms(SUFFIX) + run = [c for c in self.docker() if c.startswith("run ")][-1].split() + self.assertEqual(run[run.index("--network") + 1], "nullius-osworld2") + self.assertEqual(run[-len(self.w.NAMES):], [f"{n}.{SUFFIX}" for n in self.w.NAMES]) + + def test_a_missing_override_says_so(self): + with mock.patch.object(self.w, "OVERRIDE", os.path.join(self.tmp, "none.yml")), \ + self.assertRaises(self.w.WebsiteError) as raised: + self.w.compose(self.stack, ["config"], capture=True) + self.assertIn("this benchmark's folder is incomplete", str(raised.exception)) + + def test_no_builder_or_an_unknown_group_builds_nothing(self): + with self.assertRaises(self.w.WebsiteError): + self.w.build(self.stack, groups=["nope"]) + with mock.patch.dict(os.environ, FAKE_NO_BUILDER="1"), self.assertRaises(self.w.WebsiteError) as raised: + self.w.build(self.stack) + self.assertIn("no builder", str(raised.exception)) + self.assertFalse([c for c in self.docker() if " pull " in c or " build " in c]) + + +class Started(Exception): + pass + + +def load_driver(up): + spec = importlib.util.spec_from_file_location("osworld2_driver_websites", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py"]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +class Driver(unittest.TestCase): + """The driver's use of websites.py.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.driver = load_driver(self.tmp) + self.w = self.driver.websites + self.envs, self.checked, self.held = [], [], [] + + def run(argv, cwd, env, *a, **kw): + self.envs.append(env) + raise Started() + + @contextlib.contextmanager + def held(eid, value, closed): + self.held.append((eid, value, closed)) + yield + + for target, name, fake in ((self.driver, "_vm", lambda dir, domain, eid: os.path.join(dir, "vm.qcow2")), + (self.driver, "_run", run), (self.driver, "_task_file", lambda *a: None), + (self.w, "held", held), (self.w, "ready", lambda *a, **kw: self.checked.append((a, kw)))): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + + def option(self, value): + p = mock.patch.dict(os.environ, {self.w.OPTION: value}) + p.start() + self.addCleanup(p.stop) + + def answer(self, eid): + return self.driver.answer(f"tasks__{eid}", self.tmp, "original", "MiniMax-M3", "http://127.0.0.1:9/u1/v1", "k") + + def test_a_task_that_needs_the_sites_is_refused_without_a_suffix_before_anything_runs(self): + self.option("none") + setup = self.driver.EnvironmentSetupError + for start, kind in ((lambda: self.driver.setup("tasks__005", self.tmp), self.w.WebsiteError), + (lambda: self.answer("005"), setup), (lambda: self.driver.score("tasks__005", self.tmp), setup)): + with self.assertRaises(kind) as raised: + start() + self.assertIn(("WebsiteError: " if kind is setup else "") + "task 005 needs the mock websites", str(raised.exception)) + self.assertEqual(self.envs, []) + + def test_the_suffix_reaches_every_process_and_overleaf_is_held_around_the_run(self): + self.option(SUFFIX) + for start in (lambda: self.answer("057"), lambda: self.driver.score("tasks__057", self.tmp), + lambda: self.driver.setup("tasks__001", self.tmp)): + with self.assertRaises(Started): + start() + self.assertEqual(self.envs[-1]["WEBSITE_HOST_SUFFIX"], SUFFIX) + self.assertEqual(self.held, [("057", SUFFIX, mock.ANY)] * 2) + self.assertIsNotNone(self.held[0][-1].until) + + def test_no_suffix_is_no_variable(self): + self.option("none") + with self.assertRaises(Started): + self.answer("001") + self.assertNotIn("WEBSITE_HOST_SUFFIX", self.envs[-1]) + self.assertEqual(self.held, [("001", None, mock.ANY)]) + + def test_the_sites_are_checked_among_what_a_vm_start_needs(self): + for name in ("vm_image", "_provider_image", "_write_limit"): + p = mock.patch.object(self.driver, name, lambda: "image") + p.start() + self.addCleanup(p.stop) + made = [] + with mock.patch.object(self.driver, "_nat", lambda: True), mock.patch.object(self.driver.os, "listdir", lambda p: ["x"]), \ + mock.patch.object(self.driver.os.path, "isdir", lambda p: True), \ + mock.patch.object(self.driver._startup(), "network", + lambda docker: made.append((docker, len(self.checked))) or "nullius-osworld2"): + self.option(SUFFIX) + self.driver._ready("tasks", "001") + self.driver._ready("tasks", "081") + self.assertEqual(self.checked, [((SUFFIX,), {"state": False})]) + self.assertEqual(made, [(self.driver._asked, 0), (self.driver._asked, 0)]) + self.assertEqual(self.w.SITE, self.driver.SITE) + self.option("none") + with self.assertRaises(self.w.WebsiteError): + self.driver._ready("tasks", "081") + + def test_a_hosts_snapshot_carries_the_module_and_the_override(self): + from pathlib import Path + + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + manifest = json.loads((Path(root) / "manifest.json").read_text()) + for name in ("websites.py", "websites.compose.yml"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + self.assertIn(f"catalogue/benchmarks/osworld2/{name}", manifest["files"]) + + def test_the_override_is_the_one_the_command_names(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + command = json.load(f)["driver"]["command"] + self.assertEqual(command[-2], "{here}/websites.compose.yml") + self.assertEqual(self.w.OVERRIDE, os.path.join(HERE, "websites.compose.yml")) + given = os.path.join(self.tmp, "snapshot", "websites.compose.yml") + spec = importlib.util.spec_from_file_location("osworld2_driver_argv", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + files = [os.path.join(HERE, n) for n in ("agents.json", "nop.py", os.path.join("startup", "sitecustomize.py"))] + with mock.patch.object(sys, "argv", ["driver.py", *files, given]), \ + mock.patch.object(sys, "path", [HERE, *sys.path]), mock.patch.dict(os.environ, NULLIUS_UPSTREAM=self.tmp): + spec.loader.exec_module(driver) + self.addCleanup(setattr, driver.websites, "OVERRIDE", os.path.join(HERE, "websites.compose.yml")) + self.assertEqual(driver.websites.OVERRIDE, given) + + def test_the_lists(self): + lists = self.driver.LISTS + gitlab_or_proxy = set("026 036 050 056 098".split()) + self.assertEqual(lists["websites35"], sorted(self.w.TASKS - gitlab_or_proxy)) + self.assertEqual(lists["websites98"], sorted(lists["no_services63"] + lists["websites35"])) + self.assertEqual(len(set(lists["websites98"])), 98) + self.assertFalse(set(lists["no_services63"]) & self.w.TASKS) + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + counted = {d["id"]: d["units"] for d in bench["datasets"]} + self.assertEqual((counted["websites35"], counted["websites98"]), (35, 98)) + self.assertEqual(bench["options"]["websites"]["pattern"], "none|" + self.w.SUFFIX.pattern) + with mock.patch.dict(os.environ, {"NULLIUS_DATASET": "", "NULLIUS_DATASET_ID": "websites98"}): + self.assertEqual(len(self.driver.units()["units"]), 98) + + +class DriverSites(Case): + """Overleaf down, or reset for one of its tasks, fails no other task's check.""" + + def setUp(self): + super().setUp() + self.driver = load_driver(self.tmp) + sites = self.driver.websites + for target, name, fake in ((sites, "PROXIES", self.w.PROXIES), (sites, "_resolves", self.w._resolves), + (sites, "LOCK", self.w.LOCK), (sites, "SITES_LOCK", self.w.SITES_LOCK), + (self.driver, "_task_file", lambda *a: None), (self.driver, "_nat", lambda: True), + (self.driver, "vm_image", lambda: "image"), (self.driver, "_provider_image", lambda: None), + (self.driver, "_write_limit", lambda: None), (self.driver, "ASSETS", HERE), + (self.driver._startup(), "network", lambda docker: "nullius-osworld2")): + p = mock.patch.object(target, name, fake) + p.start() + self.addCleanup(p.stop) + p = mock.patch.dict(os.environ, {sites.OPTION: SUFFIX}) + p.start() + self.addCleanup(p.stop) + + def test_the_reaper_keeps_the_unit_s_locks_also_when_its_time_ran_out(self): + given = [] + with mock.patch.object(self.driver.subprocess, "Popen", lambda *a, **kw: given.append(kw.get("pass_fds"))): + with self.driver.websites.held("005", SUFFIX, threading.Event()): + fds = self.driver.websites.held_fds() + self.driver._reaper(1, self.tmp, True) + with self.assertRaises(RuntimeError): + self.driver._stop(mock.Mock(pid=1), os.path.join(self.tmp, "x.log"), "the run", True) + self.driver._reaper(1, self.tmp, True) + self.assertEqual(len(fds), 1) + self.assertEqual(given, [fds, fds, ()]) + + def test_overleaf_down_fails_no_other_task(self): + Sites.down.add("overleaf") + self.assertEqual(self.driver._ready("tasks", "005"), "image") + + def test_no_site_s_state_is_read_for_a_unit_s_check(self): + Sites.stateless.update(self.w.STATEFUL) + self.assertEqual(self.driver._ready("tasks", "005"), "image") + with self.assertRaises(self.w.WebsiteError): + self.w.ready(SUFFIX) + + def test_overleaf_being_reset_for_its_task_fails_no_other_task(self): + inside, released = threading.Event(), threading.Event() + + def anew(value, closed=None): + Sites.down.add("overleaf") + inside.set() + released.wait(10) + Sites.down.discard("overleaf") + + with mock.patch.object(self.driver.websites, "reset_overleaf", anew): + def first(): + with self.driver.websites.held("057", SUFFIX, threading.Event()): + pass + + t = threading.Thread(target=first) + t.start() + self.assertTrue(inside.wait(10)) + self.assertEqual(self.driver._ready("tasks", "005"), "image") + self.assertEqual(self.driver._ready("tasks", "072"), "image") + released.set() + t.join(10) + + +def _web(): + stack = os.environ.get("NULLIUS_OSWORLD2_WEB", "") + return stack if stack and shutil.which("docker") and os.path.isfile(os.path.join(stack, "docker-compose.yml")) else "" + + +@unittest.skipUnless(_web(), "NULLIUS_OSWORLD2_WEB names no OSWorld-web, or no docker compose") +class Override(unittest.TestCase): + def test_the_project_as_compose_reads_it(self): + w = load() + tiers = w.allocation(8, 80) + settings = {"HOST_SUFFIX": SUFFIX, "CADDY_SCHEME": "http://", "OSWORLD2_WEB_ADDRESS": GATEWAY, + "OSWORLD2_WEB_DISK": "/dev/sda", + **{f"OSWORLD2_WEB_{t}_{k}": str(v[i]) for t, v in tiers.items() for i, k in enumerate(("BPS", "IOPS"))}} + each, total = w.limits(settings) + self.assertLessEqual(total, (8 * w.MB, 80)) + self.assertGreater(total[0], 8 * w.MB - 1024) + said = subprocess.run(["docker", "compose", "-p", w.PROJECT, "-f", os.path.join(_web(), "docker-compose.yml"), + "-f", w.OVERRIDE, "config", "--format", "json"], capture_output=True, text=True, + env=dict(os.environ, **settings), timeout=120) + self.assertEqual(said.returncode, 0, said.stderr) + services = json.loads(said.stdout)["services"] + self.assertEqual(len(services), 40) + self.assertEqual(sorted(services), sorted(w.SERVICES)) + built = {n for n, s in services.items() if "build" in s} + self.assertEqual(built, {s for g, group, _ in w.GROUPS if g != "pulled" for s in group}) + for name, service in services.items(): + self.assertEqual(service.get("blkio_config"), { + "device_write_bps": [{"Path": "/dev/sda", "Rate": str(each[name][0])}], + "device_write_iops": [{"Path": "/dev/sda", "Rate": str(each[name][1])}]}, name) + self.assertEqual(service.get("logging"), {"driver": "json-file", "options": {"max-size": "10m", "max-file": "3"}}, name) + if name != "caddy": + self.assertNotIn("ports", service, name) + self.assertEqual([(p["host_ip"], p["published"]) for p in services["caddy"]["ports"]], [(GATEWAY, "80")]) + labelled = {v for s in services.values() for k, v in (s.get("labels") or {}).items() if k == "caddy"} + served = {h.strip()[len("http://"):-len(SUFFIX) - 1] for v in labelled for h in v.split(",")} + self.assertEqual(served, set(w.NAMES)) + self.assertEqual(served - {"studio.streamview", "interfacinglinux", "overleaf"}, set(w.STATEFUL)) + + +def _task_classes(): + up = os.environ.get("NULLIUS_OSWORLD2_UPSTREAM", "") + return sorted(glob.glob(os.path.join(up, "evaluation_examples", "task_class", "task_*.py"))) if up else [] + + +@unittest.skipUnless(_task_classes(), "NULLIUS_OSWORLD2_UPSTREAM names no upstream with the gated task classes") +class TaskClasses(unittest.TestCase): + """The tasks that import the website controller, and those that name Overleaf's site.""" + + def test_the_tasks_that_need_the_sites_and_overleaf_s(self): + w = load() + hosted, overleaf = set(), set() + for p in _task_classes(): + n = os.path.basename(p)[5:-3] + with open(p, encoding="utf-8") as f: + tree = ast.parse(f.read()) + for node in ast.walk(tree): + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [a.name for a in node.names] if isinstance(node, ast.Import) else []) + if any(m.endswith("controllers.website") for m in modules) or ( + isinstance(node, ast.ImportFrom) and (node.module or "").endswith("controllers") + and "website" in {a.name for a in node.names}): + hosted.add(n) + if isinstance(node, ast.Constant) and isinstance(node.value, str) and ( + node.value == "overleaf" or node.value.endswith("//overleaf.")): + overleaf.add(n) + self.assertEqual(sorted(hosted), sorted(w.TASKS)) + self.assertEqual(sorted(overleaf), sorted(w.OVERLEAF_TASKS)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld2_write_limit.py b/app/tests/test_osworld2_write_limit.py new file mode 100644 index 0000000..c0852b6 --- /dev/null +++ b/app/tests/test_osworld2_write_limit.py @@ -0,0 +1,554 @@ +"""OSWorld2's disk write limit: startup/sitecustomize.py on docker-py's create, and the driver's proof before a VM. +A fake docker-py, docker on PATH and sysfs stand in.""" +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +from nullius.standard import definitions, run + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HERE = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld2") +STARTUP = os.path.join(HERE, "startup") +PROVIDER = "happysixd/osworld-docker@sha256:" + "0" * 64 +OFF = {"NULLIUS_OPTION_WRITE_MBPS": "none", "NULLIUS_OPTION_WRITE_IOPS": "none", "NULLIUS_OPTION_VM_PROVIDER": "docker"} +ON = {"NULLIUS_OPTION_WRITE_MBPS": "30", "NULLIUS_OPTION_WRITE_IOPS": "300"} +MB30 = 30 * 1024 * 1024 +CONTAINERS = """class Container: + drop = False + + def __init__(self, image, kwargs): + self.image, self.kwargs, self.attrs, self.removed, self.started = image, kwargs, {}, None, False + + def start(self): + self.started = True + + def reload(self): + kept = {} if Container.drop else self.kwargs + self.attrs = {"HostConfig": {"BlkioDeviceWriteBps": kept.get("device_write_bps"), + "BlkioDeviceWriteIOps": kept.get("device_write_iops")}} + + def remove(self, **kwargs): + self.removed = kwargs + + +class ContainerCollection: + made = [] + + def __init__(self, client=None): + self.client = client + + def create(self, image, command=None, **kwargs): + ContainerCollection.made.append(Container(image, kwargs)) + return ContainerCollection.made[-1] + + def run(self, image, command=None, **kwargs): + container = self.create(image=image, command=command, **kwargs) + container.start() + return container +""" +CHILD = """import json, multiprocessing, sys +import sibling +from docker.models.containers import ContainerCollection + + +def look(queue): + run = ContainerCollection.run + queue.put({"wrapped": hasattr(ContainerCollection.create, "__wrapped__"), + "other": run(ContainerCollection(), "ubuntu:24.04", x=1).kwargs}) + + +if __name__ == "__main__": + out = {"main": hasattr(ContainerCollection.create, "__wrapped__"), "path0": sys.path[0]} + for method in ("fork", "spawn"): + ctx = multiprocessing.get_context(method) + queue = ctx.Queue() + p = ctx.Process(target=look, args=(queue,)) + p.start() + out[method] = queue.get(timeout=60) + p.join() + print(json.dumps(out)) +""" +DOCKER = """#!/bin/sh +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + info) printf '%s\t%s\n' "$FAKE_ROOT" "${FAKE_CGROUP:-2}" ;; + ps) case "$*" in *name=*) printf '%s\n' $FAKE_PROBES ;; esac ;; + create) case "$FAKE_CREATE_HANG" in + all) exec sleep 5 ;; + once) [ -e "$FAKE_DOCKER_LOG.hung" ] || { : > "$FAKE_DOCKER_LOG.hung"; exec sleep 5; } ;; + esac + [ -z "$FAKE_CREATE_FAIL" ] || { echo "Error: refused" >&2; exit 1; } + echo "WARNING: write limit discarded" >&2; echo probe ;; + inspect) echo "$FAKE_HOST" ;; + network) echo "1|10.213.0.0/24 10.213.0.1" ;; + rm) [ -z "$FAKE_RM_FAIL" ] || { echo "Error: busy" >&2; exit 1; } ;; +esac +""" + + +def load_site(environ, name="osworld2_startup"): + spec = importlib.util.spec_from_file_location(name, os.path.join(STARTUP, "sitecustomize.py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.dict(os.environ, environ): + spec.loader.exec_module(module) + return module + + +def load_driver(up, argv=()): + spec = importlib.util.spec_from_file_location("osworld2_driver_limit", os.path.join(HERE, "driver.py")) + driver = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "argv", ["driver.py", *argv]), mock.patch.object(sys, "path", [HERE, *sys.path]), \ + mock.patch.dict(os.environ, NULLIUS_UPSTREAM=up): + spec.loader.exec_module(driver) + return driver + + +def sysfs(case, site, root, kind="disk"): + """A fake sysfs and /dev for ``root``'s device: a disk, a partition, a device mapper volume, or no uevent.""" + st = os.stat(root).st_dev + tmp = tempfile.mkdtemp() + case.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + block, dev = os.path.join(tmp, "sys", "dev", "block"), os.path.join(tmp, "dev") + name = "dm-0" if kind == "dm" else "sda" + disk = os.path.join(tmp, "sys", "devices", "virtual" if kind == "dm" else "pci", name) + node = os.path.join(disk, "sda1") if kind == "partition" else disk + for d in (node, block, dev): + os.makedirs(d, exist_ok=True) + if kind != "no uevent": + with open(os.path.join(disk, "uevent"), "w") as f: + f.write(f"MAJOR=8\nMINOR=0\nDEVNAME={name}\nDEVTYPE=disk\n") + open(os.path.join(dev, name), "w").close() + if kind == "partition": + with open(os.path.join(node, "uevent"), "w") as f: + f.write("MAJOR=8\nMINOR=1\nDEVNAME=sda1\nDEVTYPE=partition\n") + with open(os.path.join(node, "partition"), "w") as f: + f.write("1\n") + open(os.path.join(dev, "sda1"), "w").close() + os.symlink(node, os.path.join(block, f"{os.major(st)}:{os.minor(st)}")) + for attr, value in (("SYS_BLOCK", block), ("DEV", dev)): + patched = mock.patch.object(site, attr, value) + patched.start() + case.addCleanup(patched.stop) + return os.path.join(dev, name) + + +class Client: + def __init__(self, root): + self.root, self.asked = root, 0 + + def info(self): + self.asked += 1 + return {"DockerRootDir": self.root} + + +class Intercept(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.root = os.path.join(self.tmp, "docker-root") + os.makedirs(self.root) + self.containers = types.ModuleType("docker.models.containers") + exec(CONTAINERS, self.containers.__dict__) + self.original = self.containers.ContainerCollection.create + modules = mock.patch.dict(sys.modules, {"docker": types.ModuleType("docker"), + "docker.models": types.ModuleType("docker.models"), + "docker.models.containers": self.containers}) + modules.start() + self.addCleanup(modules.stop) + self.site = load_site(dict(OFF, **ON)) + self.client = Client(self.root) + + def start(self, image, environ=ON, **kwargs): + self.site.wrap_docker_create(dict(OFF, **environ)) + return self.containers.ContainerCollection(self.client).run(image, **kwargs) + + def test_the_providers_container_is_limited_on_dockers_disk(self): + device = sysfs(self, self.site, self.root) + made = self.start(PROVIDER, detach=True, device_write_bps=[{"Path": "/dev/x", "Rate": 1}]) + self.assertEqual(made.image, PROVIDER) + self.assertTrue(made.kwargs["detach"]) + self.assertEqual(made.kwargs["device_write_bps"], [{"Path": "/dev/x", "Rate": 1}, {"Path": device, "Rate": MB30}]) + self.assertEqual(made.kwargs["device_write_iops"], [{"Path": device, "Rate": 300}]) + self.assertIsNone(made.removed) + self.assertTrue(made.started) + + def test_a_partition_is_limited_on_its_disk(self): + device = sysfs(self, self.site, self.root, "partition") + self.assertEqual(self.start(PROVIDER).kwargs["device_write_bps"][0]["Path"], device) + + def test_a_device_mapper_volume_is_limited_as_it_is(self): + device = sysfs(self, self.site, self.root, "dm") + self.assertEqual(device.rsplit(os.sep, 1)[1], "dm-0") + self.assertEqual(self.start(PROVIDER).kwargs["device_write_iops"], [{"Path": device, "Rate": 300}]) + + def test_one_limit_alone(self): + device = sysfs(self, self.site, self.root) + made = self.start(PROVIDER, {"NULLIUS_OPTION_WRITE_IOPS": "150"}) + made.kwargs.pop("labels") + self.assertEqual(made.kwargs, {"device_write_iops": [{"Path": device, "Rate": 150}], + "network": self.site.NETWORK}) + + def test_the_disk_is_found_once_a_process(self): + sysfs(self, self.site, self.root) + self.start(PROVIDER) + self.containers.ContainerCollection(self.client).run(PROVIDER) + self.assertEqual(self.client.asked, 1) + + def test_another_image_is_left_alone(self): + sysfs(self, self.site, self.root) + made = self.start("ubuntu:24.04", detach=True) + self.assertEqual((made.image, made.kwargs, self.client.asked), ("ubuntu:24.04", {"detach": True}, 0)) + + def test_a_container_kept_without_the_limit_is_removed_and_fails_the_start(self): + sysfs(self, self.site, self.root) + self.containers.Container.drop = True + with self.assertRaises(self.site.WriteLimitError) as raised: + self.start(PROVIDER) + made = self.containers.ContainerCollection.made[-1] + self.assertEqual((made.removed, made.started), ({"force": True, "v": True}, False)) + self.assertIn("Disk write MB/s and Disk write IOPS to none", str(raised.exception)) + + def test_no_device_fails_the_start(self): + empty = os.path.join(self.tmp, "empty") + os.makedirs(empty) + with mock.patch.object(self.site, "SYS_BLOCK", empty): + with self.assertRaises(self.site.WriteLimitError) as raised: + self.start(PROVIDER) + self.assertIn(self.root, str(raised.exception)) + self.assertEqual(self.containers.ContainerCollection.made, []) + + def test_no_uevent_fails_the_start(self): + sysfs(self, self.site, self.root, "no uevent") + with self.assertRaises(self.site.WriteLimitError): + self.start(PROVIDER) + self.assertEqual(self.containers.ContainerCollection.made, []) + + def test_no_docker_answer_fails_the_start(self): + sysfs(self, self.site, self.root) + self.client.info = lambda: {} + with self.assertRaises(self.site.WriteLimitError): + self.start(PROVIDER) + + def test_only_a_positive_whole_number_is_a_limit(self): + for value in ("30mb", "0", "007", "\u0663", "\u00b2", "-1", " 30"): + with self.subTest(value), self.assertRaises(self.site.WriteLimitError) as raised: + self.site.wanted({"NULLIUS_OPTION_WRITE_MBPS": value}) + self.assertIn("Disk write IOPS to none", str(raised.exception)) + self.assertEqual(self.site.wanted({"NULLIUS_OPTION_WRITE_MBPS": "30"}), + [("device_write_bps", MB30, "BlkioDeviceWriteBps", "--device-write-bps")]) + + def test_the_driver_and_the_wrapper_name_one_image(self): + driver = load_driver(self.tmp) + self.assertEqual(self.site.PROVIDER, driver.PROVIDER_IMAGE.split("@")[0]) + + def test_no_limit_adds_none_and_asks_docker_nothing(self): + made = self.start(PROVIDER, {}, detach=True) + self.assertEqual(sorted(made.kwargs), ["detach", "labels", "network"]) + self.assertEqual(self.client.asked, 0) + + def test_no_docker_py_changes_nothing(self): + with mock.patch.dict(sys.modules, {"docker": None, "docker.models": None, "docker.models.containers": None}): + self.site.wrap_docker_create(dict(OFF, **ON)) + self.assertIs(self.containers.ContainerCollection.create, self.original) + + def test_only_python_startup_wraps_and_only_for_docker(self): + self.assertIs(self.containers.ContainerCollection.create, self.original) + load_site(dict(ON, NULLIUS_OPTION_VM_PROVIDER="aws"), "sitecustomize") + self.assertIs(self.containers.ContainerCollection.create, self.original) + load_site(dict(ON, NULLIUS_OPTION_VM_PROVIDER="docker"), "sitecustomize") + self.assertTrue(hasattr(self.containers.ContainerCollection.create, "__wrapped__")) + + +class Processes(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + package = os.path.join(self.tmp, "fake", "docker", "models") + os.makedirs(package) + for folder in (package, os.path.dirname(package)): + open(os.path.join(folder, "__init__.py"), "w").close() + with open(os.path.join(package, "containers.py"), "w") as f: + f.write(CONTAINERS) + self.scripts = os.path.join(self.tmp, "scripts") + os.makedirs(self.scripts) + open(os.path.join(self.scripts, "sibling.py"), "w").close() + with open(os.path.join(self.scripts, "runner.py"), "w") as f: + f.write(CHILD) + + def child(self, **environ): + env = load_driver(self.tmp)._env("001", **dict(OFF, **environ)) + self.assertEqual(env["PYTHONPATH"].split(os.pathsep), [STARTUP, self.tmp]) + env["PYTHONPATH"] += os.pathsep + os.path.join(self.tmp, "fake") + ran = subprocess.run([sys.executable, os.path.join(self.scripts, "runner.py")], cwd=self.tmp, env=env, + capture_output=True, text=True, timeout=120) + self.assertEqual(ran.returncode, 0, ran.stderr) + self.assertEqual(ran.stderr, "") + return json.loads(ran.stdout) + + def test_the_limit_holds_in_the_runner_and_its_workers(self): + out = self.child(**ON) + self.assertEqual(out["path0"], self.scripts) + self.assertTrue(out["main"]) + for method in ("fork", "spawn"): + self.assertEqual(out[method], {"wrapped": True, "other": {"x": 1}}) + + def test_with_no_limit_the_create_is_still_wrapped(self): + out = self.child() + self.assertTrue(out["main"]) + for method in ("fork", "spawn"): + self.assertEqual(out[method], {"wrapped": True, "other": {"x": 1}}) + + def test_another_vm_provider_leaves_every_process_alone(self): + out = self.child(NULLIUS_OPTION_VM_PROVIDER="aws", **ON) + self.assertFalse(out["main"]) + for method in ("fork", "spawn"): + self.assertFalse(out[method]["wrapped"]) + + def test_the_file_comes_from_the_drivers_command(self): + elsewhere = os.path.join(self.tmp, "snapshot", "startup", "sitecustomize.py") + driver = load_driver(self.tmp, [os.path.join(HERE, "agents.json"), os.path.join(HERE, "nop.py"), elsewhere]) + self.assertEqual(driver._env("001")["PYTHONPATH"].split(os.pathsep)[0], os.path.dirname(elsewhere)) + + +class Before(unittest.TestCase): + """The driver's proof, before a VM start, that the limit holds.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write(DOCKER) + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + self.root = os.path.join(self.tmp, "docker-root") + os.makedirs(self.root) + self.calls = os.path.join(self.tmp, "calls") + self.driver = load_driver(self.tmp) + self.device = sysfs(self, self.driver._startup(), self.root) + self.kept = json.dumps({"BlkioDeviceWriteBps": [{"Path": self.device, "Rate": MB30}], + "BlkioDeviceWriteIOps": [{"Path": self.device, "Rate": 300}]}) + find = importlib.util.find_spec + spec = mock.patch.object(importlib.util, "find_spec", lambda n, *a: object() if n == "docker" else find(n, *a)) + spec.start() + self.addCleanup(spec.stop) + self.env = dict(OFF, **ON, PATH=bin_dir + os.pathsep + os.environ["PATH"], FAKE_DOCKER_LOG=self.calls, + FAKE_ROOT=self.root, FAKE_HOST=self.kept) + + def check(self, **environ): + with mock.patch.dict(os.environ, dict(self.env, **environ)): + self.driver._write_limit() + + def docker_calls(self): + try: + with open(self.calls) as f: + return [ln.split()[0] for ln in f] + except OSError: + return [] + + def test_a_limit_docker_keeps_passes_and_the_probe_is_removed(self): + self.check() + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "inspect", "rm"]) + with open(self.calls) as f: + create = f.read().splitlines()[2].split() + self.assertTrue(create[create.index("--name") + 1].startswith(f"nullius-osworld2-probe-{os.getuid()}-{os.getpid()}-")) + self.assertEqual(create[create.index("--device-write-bps") + 1], f"{self.device}:{MB30}") + self.assertEqual(create[create.index("--device-write-iops") + 1], f"{self.device}:300") + self.assertNotIn("start", self.docker_calls()) + + def test_a_limit_docker_drops_fails_with_what_to_do(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_HOST=json.dumps({"BlkioDeviceWriteBps": None, "BlkioDeviceWriteIOps": []})) + said = str(raised.exception) + self.assertIn("BlkioDeviceWriteBps, BlkioDeviceWriteIOps", said) + self.assertIn("write limit discarded", said) + self.assertIn("set Disk write MB/s and Disk write IOPS to none", said) + self.assertEqual(self.docker_calls()[-1], "rm") + + def test_a_failed_removal_does_not_hide_why_the_probe_failed(self): + real = self.driver._docker + + def docker(*args, **kw): + if args[0] == "rm": + raise RuntimeError("docker rm got no answer") + return real(*args, **kw) + + with mock.patch.object(self.driver, "_docker", docker), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_FAIL="1") + self.assertIn("a probe container was refused", str(raised.exception)) + + def test_a_refused_probe_fails(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_FAIL="1") + self.assertIn("refused", str(raised.exception)) + self.assertEqual(self.docker_calls()[-1], "rm") + + def test_no_disk_fails_before_any_container(self): + with mock.patch.object(self.driver._startup(), "SYS_BLOCK", os.path.join(self.tmp, "none")): + with self.assertRaises(RuntimeError): + self.check() + self.assertEqual(self.docker_calls(), ["info"]) + + def test_no_docker_py_fails(self): + with mock.patch.object(importlib.util, "find_spec", lambda n, *a: None): + with self.assertRaises(RuntimeError) as raised: + self.check() + self.assertIn("no docker-py", str(raised.exception)) + self.assertIn("install the driver's environment again", str(raised.exception)) + + def test_no_docker_root_asks_after_the_daemon(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_ROOT="") + self.assertIn("is Docker's daemon running", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info"]) + + def test_cgroup_v1_fails_before_any_container(self): + with self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CGROUP="1") + self.assertIn("cgroup v1", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info"]) + + def names(self, verb): + with open(self.calls) as f: + lines = [ln.split() for ln in f if ln.startswith(verb + " ")] + return [ln[ln.index("--name") + 1] if verb == "create" else ln[-1] for ln in lines] + + def test_every_check_waits_on_docker_twice_the_cleanups_time(self): + waited, real = [], subprocess.run + + def ran(*a, **kw): + waited.append((a[0][1], kw.get("timeout"))) + return real(*a, **kw) + + with mock.patch.object(self.driver.subprocess, "run", ran), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES="")): + self.check() + self.driver._provider_image() + self.driver._old_probes() + self.assertEqual(self.driver.READY_SECONDS, 120) + self.assertEqual(self.driver.DOCKER_SECONDS, 60) + self.assertEqual(waited, [(c, 120) for c in ("info", "ps", "create", "inspect", "rm", "image")] + [("ps", 60)]) + + def test_a_create_with_no_answer_is_tried_again_under_a_new_name(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1): + self.check(FAKE_CREATE_HANG="once") + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "create", "inspect", "rm"]) + first, second = self.names("create") + self.assertNotEqual(first, second) + self.assertEqual(self.names("rm"), [first, second]) + + def test_two_creates_with_no_answer_fail_in_one_line_and_leave_no_probe(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_HANG="all") + self.assertEqual(str(raised.exception), + "Docker's daemon did not answer docker create in 1 s: it may be busy; " + "try again or run fewer units at once") + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "create", "rm"]) + self.assertEqual(self.names("rm"), self.names("create")) + + def test_no_second_create_while_the_first_may_stand(self): + with mock.patch.object(self.driver, "READY_SECONDS", 1), self.assertRaises(RuntimeError) as raised: + self.check(FAKE_CREATE_HANG="all", FAKE_RM_FAIL="1") + self.assertIn("did not answer docker create in 1 s", str(raised.exception)) + self.assertEqual(self.docker_calls(), ["info", "ps", "create", "rm", "rm"]) + self.assertEqual(set(self.names("rm")), set(self.names("create"))) + + def test_only_this_users_gone_or_old_probes_are_removed(self): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + now, me = __import__("time").time_ns(), f"nullius-osworld2-probe-{os.getuid()}" + probes = {"dead": f"{me}-{gone.pid}-{now}", "old": f"{me}-{os.getpid()}-{now - 601 * 10 ** 9}", + "live": f"{me}-{os.getpid()}-{now}", "other": f"nullius-osworld2-probe-{os.getuid() + 1}-{gone.pid}-{now}", + "odd": f"{me}-x"} + with mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES=" ".join(probes.values()))): + self.driver._old_probes() + with open(self.calls) as f: + removed = [ln.split()[2:] for ln in f if ln.startswith("rm")] + self.assertEqual(removed, [[probes["dead"], probes["old"]]]) + + def test_no_startup_file_fails(self): + with mock.patch.object(self.driver, "SITE", os.path.join(self.tmp, "gone", "sitecustomize.py")): + with self.assertRaises(RuntimeError) as raised: + self.check() + self.assertIn("Disk write IOPS to none", str(raised.exception)) + + def test_no_limit_or_another_provider_asks_nothing(self): + self.check(**OFF) + self.check(NULLIUS_OPTION_VM_PROVIDER="aws") + self.assertEqual(self.docker_calls(), []) + + def test_the_reaper_removes_a_gone_drivers_probe(self): + gone = subprocess.Popen([sys.executable, "-c", ""]) + gone.wait() + probe = f"nullius-osworld2-probe-{os.getuid()}-{gone.pid}-{__import__('time').time_ns()}" + with mock.patch.object(self.driver, "SWEEPS", (0,)), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_PROBES=probe)): + self.driver.reap(gone.pid, self.tmp, True) + with open(self.calls) as f: + self.assertIn(f"rm -f {probe}", f.read().splitlines()) + + def test_a_vm_start_is_refused_before_the_runner_and_the_null_control(self): + starts = {"original": lambda: self.driver.answer("tasks__001", self.tmp, "original", "MiniMax-M3", + "http://relay/v1", "k"), + "nop": lambda: self.driver.score("tasks__001", self.tmp)} + for name, start in starts.items(): + with self.subTest(name), mock.patch.object(self.driver, "vm_image", lambda: "image"), \ + mock.patch.object(self.driver, "_task_file", lambda *a: None), \ + mock.patch.object(self.driver, "ASSETS", self.tmp), \ + mock.patch.object(self.driver, "_provider_image", lambda: None), \ + mock.patch.object(self.driver, "_run", side_effect=AssertionError("no process started")), \ + mock.patch.dict(os.environ, dict(self.env, FAKE_HOST="{}")): + with self.assertRaises(RuntimeError) as raised: + start() + self.assertIn("the VM's disk write limit cannot be applied", str(raised.exception)) + + +class Options(unittest.TestCase): + def test_the_options_reach_the_original_setup_and_the_null_control(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + bench = json.load(f) + given = {"write_mbps": "30", "write_iops": "300"} + for setup in ("original", "yours"): + values, bad = definitions.bench_options_of(bench, "offline36", given, setup) + self.assertEqual(bad, []) + env = run.option_env(values) + self.assertEqual((env["NULLIUS_OPTION_WRITE_MBPS"], env["NULLIUS_OPTION_WRITE_IOPS"]), ("30", "300"), setup) + values, _ = definitions.bench_options_of(bench, "offline36", None, "yours") + self.assertEqual((values["write_mbps"], values["write_iops"]), ("none", "none")) + self.assertTrue(definitions.bench_options_of(bench, "offline36", {"write_mbps": "30mb"}, "original")[1]) + + +class Turn(unittest.TestCase): + def test_the_original_setups_turn_holds_500_steps(self): + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + original = json.load(f)["setups"]["original"] + self.assertEqual(original["turn_seconds"], 86400) + self.assertGreater(original["turn_seconds"], 2 * 500 * 32) + + +class Snapshot(unittest.TestCase): + def test_a_hosts_snapshot_carries_the_startup_file(self): + from nullius.standard import payload + tmp = Path(tempfile.mkdtemp(prefix="nullius-osworld2-")) + self.addCleanup(shutil.rmtree, tmp, True) + with mock.patch.dict(os.environ, {"NULLIUS_HOME": str(tmp / "profile"), "NULLIUS_CATALOGUE": str(tmp / "catalogue")}): + _, root = payload.build(str(tmp / "cache")) + entry = Path(root) / "app" / "nullius" / "catalogue" / "benchmarks" / "osworld2" + for name in ("driver.py", "nop.py", "agents.json", "startup/sitecustomize.py"): + self.assertEqual((entry / name).read_bytes(), (Path(HERE) / name).read_bytes(), name) + manifest = json.loads((Path(root) / "manifest.json").read_text()) + self.assertIn("catalogue/benchmarks/osworld2/startup/sitecustomize.py", manifest["files"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py new file mode 100644 index 0000000..49fcc16 --- /dev/null +++ b/app/tests/test_standard_context_window.py @@ -0,0 +1,318 @@ +"""An agent's last call over the model's window ends the unit as ContextWindowExceededError, whatever followed.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from nullius.runs import classify, parse +from nullius.standard import definitions, install, job, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +WORDS = "the gated task's own sentence" +OPENROUTER = f"HTTP 400: This endpoint's maximum context length is 1000 tokens. However, you requested 1200: {WORDS}" +ANTHROPIC = f"HTTP 400: prompt is too long: 1200 tokens > 1000 maximum: {WORDS}" +EMPTY = "HTTP 400: messages: at least one message is required" +AGENT_OK = {"status": 200} +OVERFLOW = {"status": 400, "error": OPENROUTER} +SIDE_EMPTY = {"status": 400, "error": EMPTY, "side": True} +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's runner ended on its simulated user's error") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' +STOPPED = ANSWERING.replace('raise RuntimeError("the upstream\'s runner ended on its simulated user\'s error")', + 'return {"answer": "DONE"}').replace('{"reward": 0.0, "detail": {}}', + '{"reward": 0.5, "detail": {}, "stopped": "context window"}') +UNREAD = '''from nullius.standard.jsonl import serve +class VerifierError(RuntimeError): + pass +def score(unit, dir, **_): + raise VerifierError("the evaluation did not read the VM") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": lambda unit, dir, setup, model, base_url, key, **_: {"answer": "DONE"}, "score": score}) +''' +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Failure(unittest.TestCase): + def test_the_agents_overflow_outranks_the_side_call_after_it(self): + calls = [AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, agent=True), (relay.CONTEXT_ERROR, OPENROUTER)) + exc, said = relay.failure(calls, words=False, agent=True) + self.assertEqual(exc, "ContextWindowExceededError") + self.assertIn("answered 400", said) + self.assertIn("context window", said) + self.assertNotIn(WORDS, said) + self.assertNotIn("maximum context length", said) + + def test_anthropics_words_and_a_later_rate_limit(self): + calls = [AGENT_OK, {"status": 400, "error": ANTHROPIC}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_the_agents_overflow_as_the_last_call(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW], words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_an_overflow_the_agent_recovered_from_is_not_the_units(self): + self.assertIsNone(relay.failure([OVERFLOW, AGENT_OK], agent=True)) + + def test_a_side_call_that_went_through_after_it_does_not_recover_the_agent(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW, {"status": 200, "side": True}], agent=True)[0], relay.CONTEXT_ERROR) + limited = [AGENT_OK, {"status": 429, "error": "slow down"}, {"status": 502, "error": "x"}, + {"status": 429, "error": "slow down"}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(limited, words=False, agent=True)[0], "RateLimitError") + self.assertIsNone(relay.failure(limited[1:3] + [AGENT_OK, {"status": 200, "side": True}], agent=True)) + self.assertIsNone(relay.failure(limited, words=False)) + calls = [AGENT_OK, OVERFLOW, {"status": 200, "side": True}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + self.assertEqual(relay.failure(calls, words=False)[0], "RateLimitError") + calls = [{"status": 429, "error": "x"}, {"status": 502, "error": "x", "side": True}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "RateLimitError") + + def test_the_calls_are_read_in_the_order_they_began(self): + abandoned = {"status": 502, "error": "x", "t0": 1} + self.assertIsNone(relay.failure([dict(AGENT_OK, t0=2), abandoned], agent=True)) + self.assertEqual(relay.failure([{"status": 429, "error": "x", "t0": 2}, dict(AGENT_OK, t0=1)])[0], "RateLimitError") + self.assertEqual(relay.failure([dict(AGENT_OK, t0=1), {"status": 429, "error": "x"}], agent=True)[0], "RateLimitError") + + def test_another_400_of_the_agent_stays_the_infrastructures(self): + calls = [AGENT_OK, {"status": 400, "error": "HTTP 400: invalid image"}, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_side_calls_overflow_is_not_the_agents(self): + calls = [AGENT_OK, dict(OVERFLOW, side=True)] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_judges_or_simulators_route_is_never_the_agents_context(self): + calls = [AGENT_OK, OVERFLOW] + self.assertEqual(relay.failure(calls, words=False)[0], "ApiError") + self.assertEqual(relay.failure(calls)[0], "ApiError") + + def test_only_a_400_says_so(self): + self.assertTrue(relay.overflowed(OVERFLOW)) + self.assertTrue(relay.overflowed({"status": 400, "error": ANTHROPIC})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: input length and `max_tokens` exceed context " + "limit: 196000 + 8192 > 200000"})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: The input token count (1200) exceeds the " + "maximum number of tokens allowed (1000)."})) + self.assertFalse(relay.overflowed(dict(OVERFLOW, status=500))) + self.assertFalse(relay.overflowed({"status": 400, "error": EMPTY})) + self.assertFalse(relay.overflowed({"status": 400})) + + def test_it_is_the_agents_result_not_the_infrastructure(self): + self.assertEqual(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.AGENT_ERROR) + self.assertNotIn(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.INFRA_OUTCOMES) + fed = job.feed_outcome("score", None, (relay.CONTEXT_ERROR, "said")) + self.assertEqual((fed["outcome"], fed["exc"]), ("agent_error", relay.CONTEXT_ERROR)) + + +class Calls: + """The agent's calls ending on an overflow, then a refused side call.""" + def __init__(self, own, dir=os.devnull): + self.own, self.dir = own, dir + + def calls(self, tag): + return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Unit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "paper") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, setups=SETUPS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def ran(self, own, gated=True, record=os.devnull): + b = dict(self.b, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own, record)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f).get("exception_info"), parse.parse_job(job_dir) + + def test_the_unit_is_the_agents_context_said_by_status_alone(self): + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn("answered 400", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + (trial,) = ran.trials + self.assertEqual(trial.outcome, classify.AGENT_ERROR) + self.assertFalse(trial.is_infra) + self.assertEqual(ran.n_infra, 0) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY], gated=False) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn(WORDS, said["exception_message"]) + + def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW]) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + + def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_a_429(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace('"stopped": "context window"', '"stopped": "batch cap"')) + record = os.path.join(self.tmp, "record") + os.makedirs(record) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("batch cap\n") + limited = {"status": 429, "error": "slow down"} + for calls in ([AGENT_OK, limited], [AGENT_OK, limited, {"status": 502, "error": "down", "side": True}]): + said, ran = self.ran(calls, record=record) + self.assertIsNone(said, calls) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + with open(os.path.join(ran.trials[0].dir, "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + said, _ = self.ran([AGENT_OK, limited]) + self.assertEqual(said["exception_type"], "RateLimitError") + for own in (AGENT_OK, {"status": 401, "error": "no"}, {"status": 502, "error": "down"}): + said, _ = self.ran([own, dict(limited, side=True)], record=record) + self.assertEqual(said["exception_type"], "RateLimitError", own) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("context window\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + with open(os.path.join(record, relay.STOP), "a") as f: + f.write("batch cap\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace(', "stopped": "context window"', "")) + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD) + said, ran = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "VerifierError") + self.assertIn("the evaluation did not read the VM", said["exception_message"]) + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD.replace("raise VerifierError", "raise RuntimeError")) + said, _ = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + + def test_a_unit_run_again_under_its_jobs_name_starts_with_no_earlier_record_and_keeps_a_new_stop(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + record = os.path.join(jobs, "again", "a__u0000", ".work", "relay") + os.makedirs(record) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "verifier")) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "agent")) + for name, text in (("u0000.jsonl", json.dumps({"status": 429, "error": "slow down"}) + "\n"), (relay.STOP, "batch cap\n"), + ("../../verifier/score.json", "{}"), ("../../result.json", "{}"), ("../../agent/old.txt", "")): + with open(os.path.join(record, name), "w") as f: + f.write(text) + seen = [] + + def phase(r, name, unit, phase, timeout=None): + if phase == "env_setup": + os.makedirs(record, exist_ok=True) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("key cap\n") + + def unit_relay(r, tag, trial, *a, **k): + with open(os.path.join(trial, ".work", "relay", relay.STOP)) as f: + seen.append((sorted(os.listdir(os.path.join(trial, ".work", "relay"))), f.read())) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "phase", phase), patch.object(run.Run, "unit_relay", unit_relay): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + self.assertEqual(seen, [([relay.STOP], "key cap\n")]) + (earlier,) = [d for d in os.listdir(os.path.dirname(record)) if d.startswith("earlier.")] + earlier = os.path.join(os.path.dirname(record), earlier) + with open(os.path.join(earlier, "relay", relay.STOP)) as f: + self.assertEqual(f.read(), "batch cap\n") + for part in (("verifier", "score.json"), ("result.json",), ("agent", "old.txt")): + self.assertTrue(os.path.isfile(os.path.join(earlier, *part))) + self.assertFalse(os.path.exists(os.path.join(jobs, "again", "a__u0000", "verifier", "score.json"))) + + def test_a_run_under_a_job_name_another_run_holds_is_refused_before_it_writes(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + job_dir = os.path.join(jobs, "again") + lck = os.path.join(job_dir, ".work", "job.lck") + tried = subprocess.run([sys.executable, "-c", "import fcntl, sys\nfcntl.flock(open(sys.argv[1], 'a'), fcntl.LOCK_EX | fcntl.LOCK_NB)", lck], + capture_output=True) + self.assertNotEqual(tried.returncode, 0) + run.HELD.pop(lck).close() + before = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time\nf = open(sys.argv[1], 'a')\n" + "fcntl.flock(f, fcntl.LOCK_EX)\nprint(flush=True)\ntime.sleep(30)", + lck], stdout=subprocess.PIPE) + self.addCleanup(holder.stdout.close) + self.addCleanup(holder.kill) + holder.stdout.readline() + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])), \ + self.assertRaisesRegex(run.Refused, "another run under the job name again is running"): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + after = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + self.assertEqual(after, before) + + def test_a_second_run_in_the_same_process_under_a_live_job_name_is_refused(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + go = lambda: run.run(self.b, o, model="paper-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + name="live", say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + tried = [] + + def relay(r, *a, **k): + if not tried: + tried.append(None) + for again in (go, r.start): + try: + tried.append(again()) + except run.Refused as e: + tried.append(e) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "unit_relay", relay): + go() + self.assertEqual(len(tried), 3) + for refused in tried[1:]: + self.assertIsInstance(refused, run.Refused) + self.assertIn("another run under the job name live is running", str(refused)) + go() + + def test_a_side_call_failing_after_the_agents_success_stays_the_infrastructures(self): + said, ran = self.ran([AGENT_OK, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ApiError") + self.assertEqual(ran.trials[0].outcome, classify.API_ERROR) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index e1c6b84..7dd4f93 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -166,6 +166,107 @@ def test_a_standard_job_named_oracle_without_the_record_is_no_control(self): self.assertFalse(parse.parse_job(j.dir).is_control) # a runner's arm called oracle stays an arm +SLOW_SCORE = '''import os, time +from nullius.standard.jsonl import serve +CHECKED = os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "checked")) +def setup(unit, dir): + return dict({"prompt": "hi"}, **({"verify": {"run": "true"}} if CHECKED else {})) +def score(unit, dir, transcript, answer, verified=None): + time.sleep(2) + return {"reward": 0.0, "passed": False, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "score": score}) +''' + + +class Box: + image = "" + + def give_back(self, paths): + pass + + def stop(self): + pass + + +class ScoreLimit(unittest.TestCase): + """A null control whose driver does its work in ``score``, waited for ``limits.score_seconds``.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_SCORE) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def problems(self, limits): + return definitions.problems(dict(self.b, limits=limits), "benchmark") + + def trial(self, limits): + b = dict(self.b, limits=limits) + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", + jobs_root=os.path.join(self.tmp, "jobs"), env={}, say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f) + + def verify_phase(self): + jobs = os.path.join(self.tmp, "jobs") + with open(os.path.join(jobs, os.listdir(jobs)[0], "events.jsonl")) as f: + return next(e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "verify") + + def test_the_limit_is_a_whole_number_of_seconds_from_1(self): + self.assertEqual(self.problems({"score_seconds": 1800}), []) + for v in (0, -1, 1.5, "600", True, 100001): + self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + + def test_run_again_is_true_or_false(self): + for v in (False, True, None): + self.assertEqual(self.problems({"run_again": v}), [], v) + for v in (0, "no"): + self.assertIn("limits.run_again: false when a rate-limited unit is not run once more", self.problems({"run_again": v}), v) + + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): + r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) + self.assertEqual(r.score_seconds, 600) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}) + self.assertEqual(r.score_seconds, 1800) + + def test_a_score_past_the_limit_is_the_verifiers_error_and_one_within_it_scores(self): + late = self.trial({"score_seconds": 1}) + self.assertEqual(late["exception_info"]["exception_type"], "VerifierError") + self.assertIn("score: no reply in 1 s", late["exception_info"]["exception_message"]) + self.assertIsNone(late.get("verifier_result")) + on_time = self.trial({"score_seconds": 10}) + self.assertIsNone(on_time.get("exception_info")) + self.assertEqual(on_time["verifier_result"]["rewards"]["reward"], 0.0) + + def test_the_verify_phase_carries_the_score_limit_and_an_own_images_check(self): + self.trial({"score_seconds": 10}) + self.assertEqual(self.verify_phase()["timeout_sec"], 10) + shutil.rmtree(os.path.join(self.tmp, "jobs")) + open(os.path.join(self.b["_here"], "checked"), "w").close() + checked = [] + with patch.object(run.Run, "unit_environment", lambda r, setup: ("task", {"image": "own"})), \ + patch.object(run.Run, "not_run", lambda r, e, d: None), \ + patch.object(run.Run, "unit_relay", lambda r, *a, **k: None), \ + patch.object(run.Run, "own_box", lambda r, *a: Box()), \ + patch.object(run.Run, "box_changes", lambda r, box: ""), \ + patch.object(run.Run, "release_image", lambda r, image: None), \ + patch.object(run.Run, "verify", lambda r, step, box, trial: checked.append(step) or {"exit": 0}): + got = self.trial({"score_seconds": 10}) + self.assertIsNone(got.get("exception_info")) + self.assertEqual(checked, [{"run": "true"}]) + self.assertEqual(self.verify_phase()["timeout_sec"], 10 + run.TURN_SECONDS) + + class Prove(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() @@ -281,6 +382,254 @@ def test_units_that_may_not_be_redistributed_are_counted_never_named(self): self.assertIn("held-unit-2", b["proven"]["leaky"]["failed"]["leak"]["problems"][0]) +class ProveOptions(unittest.TestCase): + """The benchmark's options reach the checks' driver and each control's run; a merged control job is held as one.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "kits", "agent-task-v1") + shutil.copytree(os.path.join(definitions.KITS, "agent-task-v1"), here) + with open(os.path.join(here, "driver.py")) as f: + src = f.read() + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(src.replace('return {"units": list(WORDS)}', + 'import os\n return {"units": list(WORDS) if os.environ.get("NULLIUS_OPTION_SITES") == "on" else []}')) + with open(os.path.join(here, "standard.json")) as f: + d = json.load(f) + d["options"] = {"sites": {"summary": "The sites.", "values": ["off", "on"], "default": "off"}, + "lane": {"summary": "A lane.", "values": ["a", "b"], "default": "a"}, + "tool": {"summary": "A tool.", "values": ["none", "x"], "default": "none", + "needs": {"x": {"host": ["tool"]}}}} + d["host"] = {"tool": {"summary": "A program this machine lacks.", "check": ["nullius-no-such-program-k5"]}} + with open(os.path.join(here, "standard.json"), "w") as f: + json.dump(d, f) + self.b, bad = definitions._read(os.path.join(here, "standard.json"), "benchmark", "agent-task-v1") + assert not bad, bad + self.jobs = os.path.join(self.tmp, "jobs") + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def prove(self, **kw): + kw.setdefault("stamp", "2026-01-02__03-04-05") + return prove.prove(self.b, jobs_root=self.jobs, env={}, say=lambda s: None, **kw) + + def config(self, job_dir): + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"] + + def nop_shard(self, i, n, lane): + return run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=self.jobs, env={}, + say=lambda s: None, shard=(i, n), launch="L", bench_options={"sites": "on", "lane": lane}) + + def test_the_options_reach_the_units_driver_and_each_controls_run(self): + got = self.prove(bench_options={"sites": "on"}, write=False)["kit"] + self.assertTrue(all(r["ok"] for r in got.values()), got) + self.assertEqual((got["units"]["units"], got["nop"]["passed"], got["oracle"]["passed"]), (3, "0/3", "3/3")) + self.assertEqual(self.config(os.path.join(self.jobs, got["nop"]["job"]))["benchmark_options"], + {"sites": "on", "lane": "a", "tool": "none"}) + self.assertEqual(got["nop"]["options"], {"sites": "on", "lane": "a", "tool": "none"}) + dflt = self.prove(write=False, stamp="2026-01-02__03-04-06")["kit"] + self.assertEqual(dflt["units"]["problems"], ["units: no ids"]) + self.assertEqual(dflt["nop"]["problems"], ["the units check failed: the control was not run"]) + + def test_a_name_or_a_value_the_benchmark_does_not_allow_is_refused_before_anything_runs(self): + was = json.dumps(self.b.get("proven")) + for given, why in (({"sites": "maybe"}, "'maybe' is not one of off, on"), ({"nope": "x"}, "has no option 'nope'")): + with self.assertRaises(run.Refused) as e: + self.prove(bench_options=given) + self.assertIn(why, str(e.exception)) + self.assertFalse(os.path.exists(self.jobs)) + self.assertEqual(json.dumps(self.b.get("proven")), was) + + def test_an_option_whose_value_needs_a_program_this_machine_lacks_is_refused_before_any_install(self): + with patch.object(install, "upstream", side_effect=AssertionError("installed")) as up: + with self.assertRaises(run.Refused) as e: + self.prove(bench_options={"sites": "on", "tool": "x"}) + self.assertIn("nullius-no-such-program-k5 not on PATH", str(e.exception)) + up.assert_not_called() + self.assertFalse(os.path.exists(self.jobs)) + + def test_a_job_is_not_recorded_when_the_units_check_fails(self): + from nullius.standard import merge + whole = merge.merge([self.nop_shard(i, 4, "a") for i in (1, 3)], os.path.join(self.jobs, "nop-merged"))["out"] + self.b["datasets"][0]["units"] = 4 + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertFalse(got["units"]["ok"]) + self.assertFalse(got["nop"]["ok"]) + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", got["nop"]["problems"]) + kept = self.b["proven"]["kit"] + self.assertNotIn("nop", kept) + self.assertNotIn("units", kept) + self.assertEqual(kept["failed"]["nop"]["job"], "nop-merged") + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", + kept["failed"]["nop"]["problems"]) + + def test_a_control_is_not_run_nor_kept_when_the_units_check_fails(self): + self.assertIn("nop", self.b["proven"]["kit"]) + self.b["datasets"][0]["units"] = 4 + got = self.prove(bench_options={"sites": "on"})["kit"] + self.assertFalse(got["units"]["ok"]) + for c in ("nop", "oracle"): + self.assertEqual(got[c]["problems"], ["the units check failed: the control was not run"]) + self.assertNotIn(c, self.b["proven"]["kit"]) + self.assertIn(c, self.b["proven"]["kit"]["failed"]) + self.assertFalse(os.path.exists(self.jobs) and any("prove-" in x for x in os.listdir(self.jobs))) + + def test_a_trial_of_another_job_cannot_be_merged_into_a_proof(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, "a") for i in (1, 3)] + trial = next(t for t in sorted(os.listdir(shards[0])) if os.path.isfile(os.path.join(shards[0], t, "result.json"))) + path = os.path.join(shards[0], trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + with self.assertRaises(merge.Refused) as e: + merge.merge(shards, os.path.join(self.jobs, "never")) + self.assertIn("names another job than its job", str(e.exception)) + forced = merge.merge(shards, os.path.join(self.jobs, "forced"), force=True)["out"] + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": forced}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_a_merge_of_merges_keeps_what_any_of_them_was_merged_despite(self): + from nullius.standard import merge + good_shard, bad_shard = (self.nop_shard(i, 4, "a") for i in (1, 3)) + trial = next(t for t in sorted(os.listdir(bad_shard)) if os.path.isfile(os.path.join(bad_shard, t, "result.json"))) + path = os.path.join(bad_shard, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + good = merge.merge([good_shard], os.path.join(self.jobs, "good"))["out"] + bad = merge.merge([bad_shard], os.path.join(self.jobs, "bad"), force=True)["out"] + with self.assertRaises(merge.Refused): + merge.merge([good, bad], os.path.join(self.jobs, "never")) + both = merge.merge([good, bad], os.path.join(self.jobs, "both"), force=True)["out"] + std = self.config(both) + self.assertTrue(any(w.startswith("bad: ") and "not of the job they are in" in w for w in std["merged_despite"])) + self.assertEqual([m.get("merged_from", [{}])[0].get("job") for m in std["merged_from"]], [os.path.basename(good_shard), os.path.basename(bad_shard)]) + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": both}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_the_cli_hands_its_options_and_jobs_to_the_proof(self): + from nullius import progress + from nullius.standard import cli + seen = {} + with patch.object(cli, "_bench", return_value=self.b), patch.object(cli, "licences", return_value=(None, None)), \ + patch.object(cli, "_exit_on_term"), patch.object(progress, "ECHO", None), \ + patch.object(prove, "prove", side_effect=lambda b, **kw: seen.update(kw) or {}): + self.assertEqual(cli.main(["prove", "agent-task-v1", "--from-run", "--dataset", "kit", "--bench-option", "sites=on", + "--bench-option", "lane=b", "--job", "nop=/x/merged"]), 0) + self.assertEqual((seen["bench_options"], seen["jobs"]), ({"sites": "on", "lane": "b"}, {"nop": "/x/merged"})) + with self.assertRaises(SystemExit): + cli.main(["prove", "agent-task-v1", "--from-run", "--job", "reference=/x"]) + + def test_a_merged_launch_is_held_to_the_bound_as_one_job_and_must_cover_every_unit(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, lane) for i, lane in ((1, "a"), (3, "b"))] + whole = merge.merge(shards, os.path.join(self.jobs, "nop-merged"))["out"] + before = sorted(os.listdir(self.jobs)) + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertTrue(got["nop"]["ok"], got["nop"]) + self.assertEqual((got["nop"]["job"], got["nop"]["units"], got["nop"]["passed"]), ("nop-merged", 3, "0/3")) + self.assertEqual(self.b["proven"]["kit"]["nop"]["job"], "nop-merged") + self.assertEqual(self.b["proven"]["kit"]["nop"]["options"], + [{"lane": "a", "sites": "on", "tool": "none"}, {"lane": "b", "sites": "on", "tool": "none"}]) + self.assertEqual(sorted(set(os.listdir(self.jobs)) - set(before)), ["2026-01-02__03-04-05__agent-task-v1__kit__prove-oracle"]) + self.assertEqual([m["benchmark_options"]["lane"] for m in self.config(whole)["merged_from"]], ["a", "b"]) + half = merge.merge(shards[:1], os.path.join(self.jobs, "nop-half"))["out"] + part = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": half}, write=False, stamp="2026-01-02__03-04-06")["kit"]["nop"] + self.assertFalse(part["ok"]) + self.assertTrue(any("units have no trial" in p for p in part["problems"]), part["problems"]) + wrong = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"oracle": whole}, write=False, stamp="2026-01-02__03-04-07")["kit"]["oracle"] + self.assertIn("nop-merged: its control is 'nop', not 'oracle'", wrong["problems"]) + trial = next(t for t in sorted(os.listdir(whole)) if os.path.isfile(os.path.join(whole, t, "result.json"))) + with open(os.path.join(whole, trial, "result.json")) as f: + res = json.load(f) + res.pop("verifier_result", None) + res.pop("exception_info", None) + with open(os.path.join(whole, trial, "result.json"), "w") as f: + json.dump(res, f) + blank = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole}, write=False, + stamp="2026-01-02__03-04-08")["kit"]["nop"] + self.assertFalse(blank["ok"]) + self.assertIn("1 unit(s) ended with no reward from 0 to 1 and no error: read the job's trials", blank["problems"]) + for kw in ({"limit": 1}, {"shard": (1, 2)}): + with self.assertRaises(run.Refused): + self.prove(dataset="kit", jobs={"nop": whole}, **kw) + with self.assertRaises(run.Refused): + self.prove(jobs={"nop": os.path.join(self.tmp, "absent")}) + + +NUMERIC = {"key": "numeric", "_ref": "numeric", "label": "Numeric", "family": "agent", "needs": ["agent.task/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "score", "says": "a number from 0 to 1"}, + "controls": {"nop": {"source": "nullius", "from": "Nothing runs."}}, "datasets": [{"id": "all", "units": 3}]} + + +class HeldJob(unittest.TestCase): + """A control's job held to its bound, read from each trial's own record.""" + + def setUp(self): + from nullius.standard import job + self.tmp = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, self.tmp, True) + self.dir = os.path.join(self.tmp, "nop-job") + os.makedirs(self.dir) + nop = definitions.control(NUMERIC, "nop") + self.units = ["u1", "u2", "u3"] + job.write_config(self.dir, job="nop-job", bench=NUMERIC, other=nop, model="", provider="openai", dataset="all", + units=self.units, concurrency=1, granted=False) + results = [job.write_trial(os.path.join(self.dir, f"{u}__u{i:04d}"), job="nop-job", bench=NUMERIC, other=nop, model="", + unit=u, t0=1_760_000_000.0, t1=1_760_000_001.0, produce=(None, None), score={"reward": 0.0}, + exception=None, calls=[], trajectory=None) for i, u in enumerate(self.units)] + job.write_result(self.dir, "nop-job", 1_760_000_000.0, 1_760_000_002.0, results, 3) + + def held(self): + return prove.control_check(NUMERIC, NUMERIC["datasets"][0], "nop", limit=None, jobs_root=self.tmp, stamp="s", + provider="openai", model="", judge_model=None, env={}, concurrency=1, + say=lambda s: None, job_dir=self.dir, ids=self.units) + + def edit(self, unit, change): + trial = next(t for t in os.listdir(self.dir) if t.startswith(unit + "__")) + path = os.path.join(self.dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + change(res) + with open(path, "w") as f: + json.dump(res, f) + + def test_a_clean_job_passes(self): + r = self.held() + self.assertTrue(r["ok"], r["problems"]) + self.assertEqual((r["passed"], r["value"], r["units"]), ("0/3", 0.0, 3)) + + def test_a_reward_without_a_score_a_score_below_0_and_nan_are_no_score(self): + for value in ({"reward": 1.0}, {"score": -100}, {"score": float("nan")}, {"score": True}, {"score": "0"}): + self.edit("u2", lambda res: res["verifier_result"].update(rewards=value)) + r = self.held() + self.assertFalse(r["ok"], value) + self.assertIn("1 unit(s) ended with no score from 0 to 1 and no error: read the job's trials", r["problems"], value) + self.assertEqual(r["value"], 0.0) + + def test_a_trial_another_run_wrote_is_refused(self): + self.edit("u1", lambda res: res["standard"].update(benchmark="other", control={"name": "oracle"}, plugged="oracle")) + self.edit("u3", lambda res: res["standard"].update(job="another-job")) + r = self.held() + self.assertFalse(r["ok"]) + said = " | ".join(r["problems"]) + for k in ("benchmark", "control", "who ran it", "job"): + self.assertIn(f"whose own record names another {k} than the job", said) + + LEAKY = '''from nullius.standard.jsonl import serve import os ANSWER = "the reference answer is forty-two" diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py new file mode 100644 index 0000000..ea7893a --- /dev/null +++ b/app/tests/test_standard_gated.py @@ -0,0 +1,135 @@ +"""A gated benchmark's failing endpoint is said by its status alone, down to the trial's result.json.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius.standard import definitions, install, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +CONTROLS = {"nop": {"source": "nullius", "from": "Nothing runs; the answer is empty."}} +WORDS = "the gated task's own sentence" +FAILING = '''from nullius.standard.jsonl import serve +def score(unit, dir, transcript, answer, **_): + raise RuntimeError("the LLM grader's endpoint answered 402, and the task was scored without it") +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, "score": score}) +''' + + +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's grader said: status 402: " + %r) +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' % WORDS +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Calls: + """The unit's relay as Run.unit reads it: each route's calls, the judge's ending on a refusal with words.""" + def __init__(self, side): + self.side, self.dir = side, os.devnull + + def calls(self, tag): + if tag.endswith("-" + self.side): + return [{"status": 200}, {"status": 402, "error": f"status 402: {WORDS}"}] + return [] + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Gated(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "failing") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(FAILING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def saved(self, side, **bench): + b = dict(self.b, **bench) + jobs = os.path.join(self.tmp, "jobs-" + side + str(bench)) + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", jobs_root=jobs, env={}, + say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_refused_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.saved(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said = self.saved("judge") + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn(WORDS, said["exception_message"]) + + def answered(self, side, gated): + """drive_upstream's answer failing on its judge's or simulated user's refusal.""" + here = os.path.join(self.tmp, "paper") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + b = dict(self.b, _here=here, setups=SETUPS, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"answer-{side}-{gated}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_answer_failing_on_its_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.answered(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_an_answer_failing_on_a_benchmark_not_gated_keeps_its_words(self): + said = self.answered("judge", gated=False) + self.assertEqual(said["exception_type"], "NonZeroAgentExitCodeError") + self.assertIn(WORDS, said["exception_message"]) + + def test_the_failure_alone(self): + calls = [{"status": 429, "error": WORDS}] + self.assertEqual(relay.failure(calls), ("RateLimitError", WORDS)) + exc, said = relay.failure(calls, words=False) + self.assertEqual(exc, "RateLimitError") + self.assertNotIn(WORDS, said) + self.assertEqual(relay.failure([{"status": 404, "error": "no such model"}], words=False)[0], "ModelNotFoundError") + + def test_gated_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) + self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", + definitions.problems(dict(self.b, gated="yes"), "benchmark")) + self.assertIs(definitions.get("benchmark", "osworld2").get("gated"), True) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_parts.py b/app/tests/test_standard_parts.py index ea875b8..318af12 100644 --- a/app/tests/test_standard_parts.py +++ b/app/tests/test_standard_parts.py @@ -35,6 +35,15 @@ def trial(self, name, f1s): exception=None, calls=[], trajectory=None) return d + def test_a_stopped_score_says_so_in_the_result(self): + for name, score in (("stopped", {"reward": 0.5, "stopped": "batch cap"}), ("whole", {"reward": 0.5})): + job.write_trial(os.path.join(self.job, name), job="job", bench=BENCH, other={"key": "original", "_kind": "upstream"}, + model="m", unit=name, t0=0, t1=1, produce=(0, 1), score=score, exception=None, calls=[], trajectory=None) + with open(os.path.join(self.job, "stopped", "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + with open(os.path.join(self.job, "whole", "result.json")) as f: + self.assertNotIn("stopped", json.load(f)["verifier_result"]) + def test_the_mean_is_over_every_question(self): self.trial("conv-a", [1.0] * 4) # 4 questions, all right self.trial("conv-b", [0.0] * 12) # 12 questions, all wrong diff --git a/app/tests/test_standard_recipe_served_by.py b/app/tests/test_standard_recipe_served_by.py new file mode 100644 index 0000000..a24a99e --- /dev/null +++ b/app/tests/test_standard_recipe_served_by.py @@ -0,0 +1,180 @@ +"""The builder's Served by: a recipe's ``served_by`` (the serving providers +each model field pins, on OpenRouter) is checked as ``--served-by`` is and +compiled into it for each job that calls that model, by the name the run +sends it as; a recipe with none compiles to the command it did. The model's +providers come from OpenRouter's endpoint list, asked as the model list is.""" +from __future__ import annotations + +import io +import json +import os +import unittest +import urllib.error + +from nullius import providers +from nullius.standard import definitions, recipe + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +NOP = {"nop": {"source": "nullius", "from": "nothing said"}} + + +def defs(bench=None): + d = {k: definitions.load(k, ROOT)[0] for k in ("benchmark", "agent", "capability")} + if bench is not None: + d["benchmark"] = [bench if x["key"] == bench["key"] else x for x in d["benchmark"]] + return d + + +def fakebench(**kw): + return dict(definitions.find(definitions.load("benchmark", ROOT)[0], "fakebench"), **kw) + + +def judged(): + b = fakebench(controls=NOP) + return dict(b, score=dict(b["score"], judge={"model": "gpt-4o-mini", "says": "a judge"})) + + +def a_recipe(**kw): + r = {"version": 1, "harness": "standard", "name": "fake-0101-120000", "title": "t", "benchmark": "fakebench", "dataset": "all", + "other": "fake", "provider": "openrouter", "model": "openai/gpt-5-mini", "limit": 2, "concurrency": 2, "unsandboxed": True, + "where": "local"} + r.update(kw) + return r + + +class TheRecipe(unittest.TestCase): + def test_a_pin_compiles_to_the_option_and_shows_in_the_review(self): + out = recipe.compile(a_recipe(served_by={"model": ["deepinfra", "together/fp8"]}), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--served-by openai/gpt-5-mini=deepinfra,together/fp8", out["script"]) + model_row = next(row[2] for row in out["review"] if row[1] == "Model") + self.assertIn("model served by deepinfra, together/fp8", model_row) + + def test_each_job_is_pinned_where_it_calls_the_model(self): + """A control beside the agent calls no model of its own: its job takes no pin for the run's model; a judge + pin goes on every job of a judged benchmark, the control's included, by the name OpenRouter serves it as.""" + d = defs(judged()) + out = recipe.compile(a_recipe(others=["fake", "nop"], served_by={"model": ["deepinfra"], "judge": ["prov-j"]}), d) + self.assertEqual(out["errors"], []) + scripts = [p["script"] for p in out["parts"]] + self.assertIn("--served-by openai/gpt-5-mini=deepinfra", scripts[0]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[0]) + self.assertNotIn("openai/gpt-5-mini=deepinfra", scripts[1]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[1]) + row = next(r for r in out["review"] if r[1] == "Model")[2] + self.assertIn("judge served by prov-j", row) + + def test_a_pin_is_refused_as_the_option_is(self): + for kw, said in (({"provider": "openai", "model": "gpt-5-mini", "served_by": {"model": ["deepinfra"]}}, "needs OpenRouter"), + ({"served_by": {"model": ["bad slug"]}}, "not a serving provider's slug"), + ({"served_by": {"model": ["deepinfra", "deepinfra"]}}, "named twice"), + ({"served_by": {"agent": ["deepinfra"]}}, "Served by names"), + ({"served_by": ["deepinfra"]}, "Served by names"), + ({"served_by": {"judge": ["prov-j"]}}, "no job of this recipe calls the judge's model"), + ({"served_by": {"model": "deepinfra"}}, "Served by")): + with self.subTest(kw=kw): + out = recipe.compile(a_recipe(**kw), defs()) + self.assertTrue(out["errors"], out["errors"]) + self.assertTrue(all(e["step"] == recipe.STEP["model"] for e in out["errors"]), out["errors"]) + self.assertTrue(any(said in e["message"] for e in out["errors"]), out["errors"]) + self.assertEqual(out["script"], "") + + def test_one_model_in_two_fields_takes_one_list(self): + d = defs(judged()) + same = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["a"]}), d) + self.assertEqual(same["errors"], []) + lines = [line for line in same["script"].splitlines() if "--served-by" in line] + self.assertTrue(lines and all(line.count("--served-by") == 1 for line in lines)) # each way of starting it: once + other = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["b"]}), d) + self.assertIn("pins openai/gpt-4o-mini twice", other["errors"][0]["message"]) + + def test_a_pin_is_its_models_on_every_job_that_calls_it(self): + """The model field's model is also the judge's: the control beside the agent calls it as its judge, and its job + takes the pin too.""" + out = recipe.compile(a_recipe(others=["fake", "nop"], model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), defs(judged())) + self.assertEqual(out["errors"], []) + for part in out["parts"]: + self.assertIn("--served-by openai/gpt-4o-mini=prov-a", part["script"]) + + def test_a_list_with_an_empty_or_a_bad_slug_is_refused_in_sight(self): + for slugs in (["deepinfra", ""], ["deepinfra", "bad slug"]): + with self.subTest(slugs=slugs): + out = recipe.compile(a_recipe(served_by={"model": slugs}), defs()) + self.assertTrue(any("not a serving provider's slug" in e["message"] for e in out["errors"]), out["errors"]) + + def test_each_fields_model_is_given_as_the_run_sends_it(self): + """Like the paper, LongMemEval's gpt-4o is sent as openai/gpt-4o-2024-08-06 on OpenRouter: its providers are that + model's, and the pin is put on that name.""" + sets = definitions.find(definitions.load("benchmark")[0], "longmemeval")["datasets"] + r = a_recipe(benchmark="longmemeval", dataset=sets[0]["id"], other="", others=["original"], model_choice="paper", model="gpt-4o", + served_by={"model": ["openai"]}) + out = recipe.compile(r) + sent = out["served_models"]["model"] + self.assertEqual(sent, "openai/gpt-4o-2024-08-06") + self.assertIn(f"--served-by {sent}=openai", out["script"]) + self.assertEqual(recipe.compile(a_recipe(), defs())["served_models"]["model"], "openai/gpt-5-mini") + + def test_a_recipe_with_no_pin_compiles_to_the_command_it_did(self): + plain = recipe.compile(a_recipe(), defs()) + self.assertEqual(plain["errors"], []) + self.assertNotIn("--served-by", plain["script"]) + for none in ({}, {"model": []}, None): + with self.subTest(none=none): + self.assertEqual(recipe.compile(a_recipe(served_by=none), defs())["script"], plain["script"]) + openai = recipe.compile(a_recipe(provider="openai", model="gpt-5-mini", served_by={}), defs()) + self.assertEqual(openai["errors"], []) + + +class TheProviders(unittest.TestCase): + """``providers.list_endpoints``: OpenRouter's endpoint list, a fake opener in the place of the network.""" + + def opener(self, body=None, error=None): + seen = [] + + def opener(req, timeout=None): + seen.append(req) + if error: + raise error + return io.BytesIO(json.dumps(body).encode()) + return opener, seen + + def test_a_models_providers_by_slug_and_name(self): + body = {"data": {"id": "meta-llama/llama-3.3-70b-instruct", "endpoints": [ + {"provider_name": "DeepInfra", "tag": "deepinfra/fp8", "quantization": "fp8"}, + {"provider_name": "Together", "tag": "together"}, + {"provider_name": "Google AI Studio"}, # no tag: its name as a slug reads + {"provider_name": "Bad", "tag": "bad slug"}, "not a row", {"tag": 7}]}} + opener, seen = self.opener(body) + got = providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + self.assertEqual(got, [{"slug": "deepinfra/fp8", "name": "DeepInfra", "quantization": "fp8"}, + {"slug": "google-ai-studio", "name": "Google AI Studio", "quantization": ""}, + {"slug": "together", "name": "Together", "quantization": ""}]) + (req,) = seen + self.assertEqual(req.full_url, "https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints") + self.assertIsNone(req.get_header("Authorization")) # no key set: the list is asked without one + self.assertEqual(req.get_method(), "GET") + + def test_the_key_goes_as_for_the_model_list_and_a_bare_name_gets_its_maker(self): + opener, seen = self.opener({"data": {"endpoints": []}}) + providers.list_endpoints("openrouter", "gpt-4o-mini", env={"OPENROUTER_API_KEY": "or-test"}, opener=opener) + self.assertEqual(seen[0].full_url, "https://openrouter.ai/api/v1/models/openai/gpt-4o-mini/endpoints") + self.assertEqual(seen[0].get_header("Authorization"), "Bearer or-test") + + def test_what_cannot_be_asked_or_read_is_said_in_fixed_words(self): + for kw in ({"error": urllib.error.URLError("refused at https://openrouter.ai?key=x")}, {"body": {"error": "no"}}, + {"body": {"data": {"endpoints": "none"}}}): + with self.subTest(kw=kw): + opener, _ = self.opener(**kw) + with self.assertRaisesRegex(ValueError, r"^Provider list unavailable\. Enter a provider's slug by hand\.$"): + providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + opener, seen = self.opener({}) + with self.assertRaisesRegex(ValueError, "Only OpenRouter"): + providers.list_endpoints("openai", "gpt-4o", env={"OPENAI_API_KEY": "x"}, opener=opener) + for model in ("no-maker", "a/b/../../c", "maker/model?x=1", ""): + with self.subTest(model=model), self.assertRaisesRegex(ValueError, "maker/model"): + providers.list_endpoints("openrouter", model, env={}, opener=opener) + self.assertEqual(seen, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 8319794..d6748e4 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -3,6 +3,7 @@ A driver or capability process answers one JSON line per request.""" from __future__ import annotations +import base64 import http.client import json import os @@ -34,6 +35,22 @@ b'{"type":"message","content":[{"type":"output_text","text":"hi"}]}],' b'"usage":{"input_tokens":20,"output_tokens":5,"input_tokens_details":{"cached_tokens":10}}}}\n\n') +RESP_FAILED = json.dumps({"object": "response", "id": "gen-1", "status": "failed", "output": [], "usage": None, + "error": {"code": "server_error", "message": "the request said x"}}).encode() +RESP_FAILED_SSE = (b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","output":[],' + b'"error":{"code":"server_error","message":"the request said x"}}}\n\n') +RESP_INCOMPLETE = json.dumps({"object": "response", "status": "incomplete", "output": [], + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 30, "output_tokens": 8}}).encode() + +HOLLOW = {"hollow": json.dumps({"id": "gen-1", "provider": "Minimax", "model": "m"}).encode(), + "hollow-error": json.dumps({"id": "gen-1", "error": {"code": 502, "message": "the request said x"}}).encode(), + "hollow-429": json.dumps({"id": "gen-1", "error": {"code": 429, "message": "the request said x"}}).encode(), + "hollow-402": json.dumps({"id": "gen-1", "error": {"code": 402, "message": "the request said x"}}).encode(), + "hollow-404": json.dumps({"id": "gen-1", "error": {"code": 404, "message": "the request said x"}}).encode(), + "count": json.dumps({"input_tokens": 12}).encode(), "no-choice": json.dumps({"choices": []}).encode(), + "not-json": b"the request said x"} + class Upstream(BaseHTTPRequestHandler): seen = [] @@ -51,6 +68,23 @@ def do_POST(self): self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") elif mode == "no-usage": self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") + elif mode in ("400", "413", "422"): + self.reply(int(mode), b'{"error":{"message":"could not process image"}}', "application/json") + elif mode in ("cut", "short"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header(*(("Transfer-Encoding", "chunked") if mode == "cut" else ("Content-Length", str(len(CHAT_SSE) + 9)))) + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(CHAT_SSE), CHAT_SSE) if mode == "cut" else CHAT_SSE) + elif mode in HOLLOW: + self.reply(200, HOLLOW[mode], "application/json", *[("X-Generation-Id", "gen-1")] * (mode == "not-json"), + ("Retry-After", "30")) + elif self.path.endswith("/responses") and mode == "failed": + self.reply(200, RESP_FAILED, "application/json") + elif self.path.endswith("/responses") and mode == "failed-sse": + self.reply(200, RESP_FAILED_SSE, "text/event-stream") + elif self.path.endswith("/responses") and mode == "incomplete": + self.reply(200, RESP_INCOMPLETE, "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -58,10 +92,11 @@ def do_POST(self): else: self.reply(200, json.dumps(CHAT).encode(), "application/json") - def reply(self, status, body, kind): + def reply(self, status, body, kind, *headers): try: self.send_response(status) - self.send_header("Content-Type", kind) + for header in (("Content-Type", kind),) + headers: + self.send_header(*header) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -90,6 +125,7 @@ def post(self, tag, path, body): with urllib.request.urlopen(req) as r: return r.status, r.read() except urllib.error.HTTPError as e: + self.retry_after = e.headers.get("Retry-After") return e.code, e.read() def test_plain_reply_passes_through_and_the_real_key_goes_out(self): @@ -120,6 +156,68 @@ def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): self.assertEqual(rec["reply"]["text"], "hello") self.assertEqual(rec["reply"]["usage"]["input"], 7) + def test_a_stream_cut_off_after_its_headers_is_a_failed_call(self): + for mode in ("cut", "short"): + self.post("u" + mode, "/chat/completions", {"model": "m", "stream": True, "mode": mode, "messages": []}) + rec = self.relay.calls("u" + mode)[-1] + self.assertEqual(rec["status"], 502) + self.assertIn("cut off", rec["error"]) + self.assertEqual(rec["reply"]["usage"]["input"], 7) + self.assertEqual(job._usage_total([rec])["cost"], rec["cost_usd"]) + + def test_a_refused_call_keeps_each_inline_image_by_its_type_wholeness_and_size_alone(self): + png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20 + b"IEND\xaeB`\x82\n").decode() + cut = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20).decode() + jpeg = base64.b64encode(b"\xff\xd8\xff" + b"y" * 9 + b"\xff\xd9tail").decode() + webp = base64.b64encode(b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"z" * 4 + b"\n").decode() + image = lambda kind, data: {"type": "image", "source": {"type": "base64", "media_type": kind, "data": data}} + content = [image("image/png", png), image("image/png", cut), image("image/gif", "not base64!"), + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg}"}}, + {"type": "input_image", "image_url": f"data:image/webp;base64,{webp}"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "grade"}] + sent = [{"type": "image/png", "is": "image/png", "bytes": len(png)}, {"type": "image/png", "is": None, "bytes": len(cut)}, + {"type": "image/gif", "is": None, "bytes": 11}, {"type": "image/jpeg", "is": "image/jpeg", "bytes": len(jpeg)}, + {"type": "image/webp", "is": "image/webp", "bytes": len(webp)}] + for mode in ("400", "413", "422", "429", None): + self.post("img", "/messages", {"model": "m", "mode": mode, "messages": [{"role": "user", "content": content}]}) + rec = self.relay.calls("img")[-1] + with self.subTest(mode): + self.assertEqual(rec.get("images"), sent if mode in ("400", "413", "422") else None) + self.assertFalse(any(data in json.dumps(rec) for data in (png, cut, jpeg, webp))) + + def test_a_200_to_a_model_call_with_no_reply_is_a_502_said_by_its_kind_alone(self): + for mode, path, code, said, kind in ( + ("hollow", "/messages", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/chat/completions", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/responses", 502, "no reply", "NetworkConnectionError"), + ("hollow-error", "/messages", 502, "an error 502", "NetworkConnectionError"), + ("hollow-429", "/messages", 429, "an error 429", "RateLimitError"), + ("hollow-404", "/completions", 502, "an error 404", "NetworkConnectionError"), + ("hollow-402", "/messages", 402, "an error 402", "ApiError"), + ("not-json", "/messages", 502, "no reply", "NetworkConnectionError")): + tag = f"uh-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + rec = self.relay.calls(tag)[-1] + self.assertEqual(self.retry_after, "30") + with self.subTest(mode=mode, path=path): + self.assertEqual((status, rec["status"]), (code, code)) + self.assertEqual(json.loads(body)["error"]["message"], f"the endpoint answered 200 with {said}") + self.assertEqual(rec["error"], f"HTTP {code}: the endpoint answered 200 with {said}") + self.assertEqual(rec["answered"], 200) + self.assertFalse({"reply", "cost_usd"} & set(rec)) + self.assertNotIn("the request said", json.dumps(rec)) + self.assertEqual(relay.failure([rec], agent=True)[0], kind) + self.assertEqual(self.post("uk-asked-stream", "/messages", {"model": "m", "mode": "hollow", "stream": True, + "messages": []}), (200, HOLLOW["hollow"])) + for mode, path in (("count", "/messages/count_tokens"), ("no-choice", "/chat/completions"), ("hollow", "/models"), + ("hollow", "/threads/t1/messages")): + tag = f"uk-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + with self.subTest(path): + self.assertEqual((status, body), (200, HOLLOW[mode])) + self.assertEqual(self.relay.calls(tag)[-1]["status"], 200) + self.assertNotIn("answered", self.relay.calls(tag)[-1]) + def test_streamed_responses_read_from_the_completed_event(self): status, body = self.post("u3", "/responses", {"model": "m", "stream": True, "input": "x"}) self.assertEqual(body, RESP_SSE) @@ -128,6 +226,25 @@ def test_streamed_responses_read_from_the_completed_event(self): self.assertEqual(rec["reply"]["tool_calls"][0]["name"], "shell") self.assertEqual(rec["reply"]["usage"]["cached"], 10) + def test_a_failed_response_with_no_usage_costs_nothing(self): + for tag, mode in (("uf1", "failed"), ("uf2", "failed-sse")): + status, _ = self.post(tag, "/responses", {"model": "m", "mode": mode, "input": "x", "stream": mode == "failed-sse"}) + self.assertEqual(status, 200) + rec = self.relay.calls(tag)[-1] + self.assertEqual((rec["reply"]["end"], rec["reply"]["error"]["code"]), ("failed", "server_error")) + self.assertEqual((rec["cost_usd"], rec["cost_from"]), (0.0, "failed")) + failed = {"status": 200, "cost_usd": 0.0, "cost_from": "failed"} + self.assertEqual(job._usage_total([failed, {"status": 200}])["cost"], None) + self.assertEqual({k: job._usage_total([failed, {"status": 200, "cost_usd": 0.5, "cost_from": "provider"}])[k] + for k in ("cost", "cost_from")}, {"cost": 0.5, "cost_from": "provider"}) + + def test_an_incomplete_response_is_priced_from_its_usage(self): + self.post("uf3", "/responses", {"model": "m", "mode": "incomplete", "input": "x"}) + rec = self.relay.calls("uf3")[-1] + self.assertTrue(rec["reply"]["end"].startswith("incomplete")) + self.assertEqual(rec["cost_from"], "estimate") + self.assertGreater(rec["cost_usd"], 0) + def test_calls_are_recorded_by_their_unit(self): self.post("ua", "/chat/completions", {"model": "m", "messages": []}) self.post("ub", "/chat/completions", {"model": "m", "messages": []}) @@ -230,6 +347,54 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_stream_cut_off_or_a_failed_response_with_no_usage_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + for tag, path, body in (("uc", "/chat/completions", {"model": "m", "stream": True, "mode": "cut", "messages": []}), + ("uf", "/responses", {"model": "m", "mode": "failed", "input": "x"})): + req = urllib.request.Request(r.url(tag) + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,), (failed,) = r.calls("uc"), r.calls("uf") + self.assertEqual((cut["status"], failed["cost_from"]), (502, "failed")) + self.assertIsNotNone(cut.get("cost_usd")) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], cut["booked_usd"] + failed["booked_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_stream_cut_off_that_already_cost_more_than_its_booking_counts_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "stream": True, "mode": "cut", "messages": []} + req = urllib.request.Request(r.url("uc") + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with mock.patch.object(ledger, "bound", lambda *a, **k: 1e-9): + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,) = r.calls("uc") + got = led.state() + self.assertEqual((cut["status"], got["unsettled_usd"]), (502, 0.0)) + self.assertGreater(cut["cost_usd"], cut["booked_usd"]) + self.assertAlmostEqual(got["spent_usd"], cut["cost_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_unit_asked_to_stop_books_nothing_and_a_200_with_no_reply_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + with open(os.path.join(r.dir, relay.STOP), "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(r, "u1", {"model": "m", "messages": []})[0], 400) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"], got["unsettled_usd"]), (0.0, 0, 0.0)) + os.unlink(os.path.join(r.dir, relay.STOP)) + self.assertEqual(self.post(r, "u1", {"model": "m", "mode": "hollow", "messages": []})[0], 502) + (call,) = r.calls("u1") + got = led.state() + self.assertEqual((call["answered"], got["spent_usd"], got["held_usd"]), (200, 0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], call["booked_usd"]) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply (2 output tokens here) a call that names no limit is booked those, at the same price.""" @@ -451,6 +616,11 @@ def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): self.assertEqual(relay._request_tail({"input": [], "reasoning": {"effort": "high"}})["reasoning_effort"], "high") self.assertIsNone(relay._request_tail({"messages": []})["reasoning_effort"]) + def test_reasoning_is_read_as_reasoning_or_thinking_tokens(self): + self.assertEqual(relay._usage({"completion_tokens_details": {"reasoning_tokens": 3}})["reasoning"], 3) + self.assertEqual(relay._usage({"output_tokens_details": {"reasoning_tokens": 5}})["reasoning"], 5) + self.assertEqual(relay._usage({"output_tokens": 9, "output_tokens_details": {"thinking_tokens": 7}})["reasoning"], 7) + class ServeTest(unittest.TestCase): """The relay as a process of its own: the key from its environment only, diff --git a/app/tests/test_standard_relay_pin.py b/app/tests/test_standard_relay_pin.py new file mode 100644 index 0000000..aa8333d --- /dev/null +++ b/app/tests/test_standard_relay_pin.py @@ -0,0 +1,317 @@ +"""A pinned route's failure on top of the serving pin (``serving``): a pinned call no endpoint took, or one its reply +says another provider served, fails the unit as its route's. Before a fake OpenRouter.""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import tempfile +import threading +import unittest +from unittest import mock +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from nullius.runs import classify +from nullius.standard import definitions, install, job, relay, run + +M3, SONNET = "minimax/minimax-m3", "anthropic/claude-sonnet-4.6" +PINS = {M3: ["minimax"], SONNET: ["anthropic"]} +SERVES = {M3: ["Parasail", "MiniMax"], SONNET: ["Google", "Anthropic"]} +SECRET = "the words of a gated task" +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") + + +class OpenRouter(BaseHTTPRequestHandler): + """Routes as OpenRouter documents them: ``down`` a 503, no endpoint of the pin a 404.""" + ignore_pin = False + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model, pref = body.get("model"), body.get("provider") or {} + allowed = [e for e in SERVES.get(model, ["Someone"]) + if OpenRouter.ignore_pin or not pref.get("order") or e.lower() in pref["order"]] + if body.get("down"): + allowed = [] + if not allowed: + status = 503 if body.get("down") else 404 + return self.reply(status, {"error": {"code": status, "message": f"No allowed providers are available for the selected model: {SECRET}"}}) + if self.path.endswith("/messages"): + return self.reply(200, {"id": "gen-555", "type": "message", "role": "assistant", "content": [{"type": "text", "text": SECRET}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) + self.reply(200, {"id": "gen-123", "provider": allowed[0], "choices": [{"message": {"role": "assistant", "content": SECRET}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": 0.0001}}) + + def reply(self, status, out): + body = json.dumps(out).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + try: + return e.code, e.read() + finally: + e.close() + + +class Pinned(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.relay = self.pinned(PINS) + + def pinned(self, serving, model=M3): + r = relay.Relay(self.base, "sk-real", tempfile.mkdtemp(dir=self.dir), model=model, + side_models=["claude-sonnet-4-6=" + SONNET], serving=serving).start() + self.addCleanup(r.stop) + return r + + def test_no_endpoint_of_the_pin_is_the_routes_error_not_an_empty_answer(self): + r = self.pinned({M3: ["fireworks"]}) + status, _ = post(r.url("u3") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertEqual(status, 404) + kind, said = relay.failure(r.calls("u3"), serving={M3: ["fireworks"]}) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to fireworks", said) + self.assertEqual(classify.from_verdict(kind, None), classify.API_ERROR) + + def test_a_pinned_provider_down_is_the_routes_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + (call,) = self.relay.calls("u1") + self.assertEqual(call["status"], 503) + self.assertEqual(relay.failure(self.relay.calls("u1"), serving=PINS)[0], "ApiProviderRouteError") + + def test_a_gated_benchmark_keeps_the_endpoints_words_out_of_a_route_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + kind, said = relay.failure(self.relay.calls("u1"), words=False, serving=PINS) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to minimax", said) + self.assertIn("answered 503", said) + self.assertNotIn(SECRET, said) + self.assertNotIn("No allowed providers", said) + + def test_an_unpinned_404_is_still_what_it_was(self): + calls = [{"status": 404, "model": "other/model", "error": "HTTP 404: model not found"}] + self.assertEqual(relay.failure(calls, serving=PINS)[0], "ModelNotFoundError") + self.assertIsNone(relay.route_failure(calls, PINS)) + + def test_a_side_models_refusal_fails_the_unit_even_when_the_agent_answered_last(self): + unit = self.relay.url("u1") + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + calls = self.relay.calls("u1") + self.assertIsNone(relay.failure(calls, serving=PINS)) + kind, said = relay.route_failure(calls, PINS, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn(SONNET, said) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": []}) + self.assertIsNone(relay.route_failure(self.relay.calls("u1"), PINS)) + + def test_a_routes_calls_are_read_in_the_order_they_began(self): + pin, refused = {M3: ["fireworks"]}, {"model": M3, "status": 404} + self.assertIsNone(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, dict(refused, t0=1)], pin)) + self.assertEqual(relay.route_failure([dict(refused, t0=2), {"model": M3, "status": 200, "t0": 1}], pin)[0], + "ApiProviderRouteError") + self.assertEqual(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, refused], pin)[0], "ApiProviderRouteError") + + def test_a_judges_answer_does_not_cover_the_simulated_users_refusal_of_the_same_model(self): + unit, judge = self.relay.url("u1"), self.relay.url("u1-judge", model=SONNET) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + post(judge + "/messages", {"model": SONNET, "messages": []}) + routes = [self.relay.calls("u1"), self.relay.calls("u1-judge"), []] + self.assertEqual(relay.routes_failure(routes, PINS, words=False)[0], "ApiProviderRouteError") + self.assertIsNone(relay.routes_failure([self.relay.calls("u1-judge")], PINS)) + + def test_only_a_route_with_no_endpoint_stops_the_run(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + self.assertFalse(relay.route_dead(relay.failure(self.relay.calls("u1"), serving=PINS))) + r = self.pinned({M3: ["fireworks"]}) + post(r.url("u5") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertTrue(relay.route_dead(relay.failure(r.calls("u5"), words=False, serving={M3: ["fireworks"]}))) + self.assertFalse(relay.route_dead(("ApiError", "404: x"))) + + def test_an_answer_from_outside_the_pin_fails_the_unit_as_its_route(self): + pin = {"openai/gpt-5-mini": ["minimax"]} + r = self.pinned(pin, model="openai/gpt-5-mini") + SERVES["openai/gpt-5-mini"] = ["Parasail"] + self.addCleanup(SERVES.pop, "openai/gpt-5-mini") + OpenRouter.ignore_pin = True + self.addCleanup(setattr, OpenRouter, "ignore_pin", False) + post(r.url("u4") + "/chat/completions", {"model": "x", "messages": []}) + (call,) = r.calls("u4") + self.assertEqual((call["status"], call["reply"]["served_by"]), (200, "Parasail")) + self.assertIsNone(relay.failure(r.calls("u4"), serving=pin)) + kind, said = relay.route_failure(r.calls("u4"), pin, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("served by Parasail", said) + self.assertNotIn(SECRET, said) + self.assertIsNone(relay.route_failure(r.calls("u4"), {})) + + def test_a_provider_is_off_its_pin_only_when_its_name_shares_nothing_with_the_pin(self): + self.assertIs(relay.held("MiniMax", ["minimax"]), True) + self.assertIs(relay.held("Anthropic", ["anthropic"]), True) + self.assertIs(relay.held("Google AI Studio", ["google-ai-studio"]), True) + self.assertIs(relay.held("Parasail", ["minimax"]), False) + self.assertIsNone(relay.held("Google", ["google-vertex"])) + self.assertIsNone(relay.route_failure([{"status": 200, "model": "m", "reply": {"served_by": "Google"}}], {"m": ["google-vertex"]})) + + def test_a_name_that_is_not_a_providers_name_fails_nothing_and_is_never_said(self): + odd = [{"status": 200, "model": "m", "reply": {"served_by": "Parasail\n" + SECRET}}] + self.assertIsNone(relay.route_failure(odd, {"m": ["minimax"]}, words=False)) + + +def osworld2() -> dict: + benches, _ = definitions.load("benchmark") + return next(b for b in benches if b["key"] == "osworld2") + + +class Definition(unittest.TestCase): + def test_osworld2s_original_pins_m3_to_minimax_and_sonnet_to_anthropic_on_openrouter_only(self): + b = osworld2() + self.assertEqual(definitions.problems(b, "benchmark"), []) + other, why = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.assertEqual((why, other["pins"]), ("", PINS)) + other, _ = definitions.setup(b, "original", "MiniMax-M3", "custom") + self.assertNotIn("pins", other) + + def test_the_run_pins_the_official_judge_but_not_one_asked_in_its_place(self): + b = osworld2() + other, _ = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + self.assertEqual(run.pins_of(b, other, "openrouter", SONNET), PINS) + self.assertEqual(run.pins_of(b, other, "openrouter", "openai/gpt-5"), PINS) + self.assertEqual(run.pins_of(b, other, "custom", SONNET), {}) + agent = {"_kind": "agent", "key": "codex", "pins": {M3: ["x"]}} + self.assertEqual(run.pins_of(b, agent, "openrouter", SONNET), {SONNET: ["anthropic"]}) + self.assertEqual(run.pins_of(b, agent, "openrouter", "openai/gpt-5"), {}) + + def test_a_benchmark_without_pins_runs_as_before(self): + benches, _ = definitions.load("benchmark") + for b in benches: + if b["key"] == "osworld2": + continue + with self.subTest(b["key"]): + self.assertEqual(run.pins_of(b, {"_kind": "agent"}, "openrouter", "m"), {}) + for which in (b.get("setups") or {}): + if which == "original": + other, _ = definitions.setup(b, which, "", "openrouter") + self.assertNotIn("pins", other or {}) + + def test_the_definitions_pin_holds_whatever_served_by_says(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + make = lambda **kw: run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", + env={"OPENROUTER_API_KEY": "k"}, unsandboxed=True, jobs_root=tmp, say=lambda s: None, **kw) + self.assertEqual(make().serving, {SONNET: ["anthropic"]}) + r = make(served_by={"m-1": ["prov-a"], SONNET: ["google-vertex"]}) + self.assertEqual((r.serving, r.route_pins), ({"m-1": ["prov-a"], SONNET: ["anthropic"]}, {SONNET: ["anthropic"]})) + + def test_a_pin_is_checked(self): + b = osworld2() + + def problems(**orig): + d = dict(b, setups=dict(b["setups"], original=dict(b["setups"]["original"], **orig))) + return [p for p in definitions.problems(d, "benchmark") if "pin" in p] + + self.assertEqual(problems(), []) + self.assertTrue(problems(pin={"openai": {"MiniMax-M3": ["minimax"]}})) + self.assertTrue(problems(pin={"openrouter": {"GPT-9": ["minimax"]}})) + self.assertTrue(problems(pin={"openrouter": {"MiniMax-M3": ["Mini Max"]}})) + self.assertTrue(problems(pin={"openrouter": {"MiniMax-M3": []}})) + judge = dict(b["score"]["judge"], pin={"openrouter": "anthropic"}) + self.assertTrue([p for p in definitions.problems(dict(b, score=dict(b["score"], judge=judge)), "benchmark") + if "score.judge.pin" in p]) + + +class Recorded(unittest.TestCase): + def test_the_jobs_config_says_each_models_serving_pin(self): + b = osworld2() + other, _ = definitions.setup(b, "original", "MiniMax-M3", "openrouter") + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, ignore_errors=True) + bench = dict(b, units=other["units"]) + job.write_config(d, job="j", bench=bench, other=other, model=M3, provider="openrouter", dataset="test_v2", + units=["007"], concurrency=1, granted=False, judge=SONNET, served_by=run.pins_of(b, other, "openrouter", SONNET)) + with open(os.path.join(d, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {SONNET: ["anthropic"], M3: ["minimax"]}) + + +class Wired(unittest.TestCase): + """A unit of a run whose judge the definition pins and whose own model the user pins (``--served-by``).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(self.tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + self.r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "k"}, + unsandboxed=True, jobs_root=self.tmp, say=lambda s: None, served_by={"m-1": ["prov-a"]}) + + def ended(self, routes): + rr = mock.Mock(mcp_listeners=[], unit_image="", dir=self.tmp) + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 0} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit"): + stack.enter_context(mock.patch.object(self.r, name)) + self.noted = stack.enter_context(mock.patch.object(self.r, "note_refusal")) + stack.enter_context(mock.patch.object(self.r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(self.r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(self.r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = self.r.unit(0, "good") + return (res.get("exception_info") or {}).get("exception_type") + + def test_only_a_route_the_definition_pins_fails_the_unit_as_its_route(self): + ok, no = {"model": "m-1", "status": 200}, {"status": 404, "error": "HTTP 404: no endpoint"} + self.assertNotIn(self.ended({"u0000": [dict(no, model="m-1")]}), (None, "ApiProviderRouteError")) + self.assertIsNone(self.ended({"u0000": [dict(ok, reply={"served_by": "Parasail"})]})) + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertTrue(self.noted.call_args[0][0][1].startswith("404: ")) + self.assertEqual(self.ended({"u0000": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [{"model": SONNET, "status": 200, "reply": {"served_by": "Parasail"}}]}), + "ApiProviderRouteError") + + def test_three_units_in_a_row_with_no_endpoint_on_a_pinned_route_stop_the_run(self): + self.r._halt, self.r._refused, self.r.halted = threading.Event(), [], None + for _ in range(3): + self.r.note_refusal(("ApiProviderRouteError", "404: no endpoint")) + self.assertTrue(self.r._halt.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec26..4defc5c 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -624,6 +624,35 @@ def unit(*a): got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_benchmark_that_says_so_runs_no_rate_limited_unit_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}], + limits={"run_again": False}) + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + real = r.unit + + def unit(*a): + res = real(*a) + if res and res["trial_name"].startswith("model__"): + with open(os.path.join(r.job_dir, res["trial_name"], ".work", "relay", relay.STOP), "w") as f: + f.write("batch cap\n") + return res + r.unit = unit + job_dir = r.start() + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py new file mode 100644 index 0000000..fe861a2 --- /dev/null +++ b/app/tests/test_standard_served_by.py @@ -0,0 +1,768 @@ +"""A pin of the serving providers that may answer a model (``--served-by``): +the relay puts OpenRouter's routing field on every request it sends as that +model and on no other, in place of the caller's own; each call records the +routing sent, the provider that served the reply, how the reply ended and an +error inside a success, as cleaned copies, read apart from the rest of the +record; the job and each trial say who served the calls of every route and +whether the pin held; ``merge`` refuses shards of different pins; a +container's relay is given the pin as the unit's own relay is. A run without +the option sends the same requests and ends its units as it did. Every +upstream here is a fake on this machine's loopback: no request leaves it.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import select +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +import urllib.request +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import progress +from nullius.standard import cli, container, definitions, install, job, merge, relay, run + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIN = {"order": ["prov-a", "prov-b/fp8"], "allow_fallbacks": False} +KEY = "or-test-relay-key-0123456789" +NEW = ("served_by", "end", "error") # what the record's reply gains, beside what it held + + +def sse(*events) -> bytes: + return b"".join(b"data: " + json.dumps(e).encode() + b"\n\n" for e in events) + b"data: [DONE]\n\n" + + +def routing_facts(selected, *others) -> dict: + """OpenRouter's routing facts (``openrouter_metadata``) as a reply holds them when asked: the endpoint that + served it is the one marked ``selected`` among the available ones.""" + available = [{"provider": o, "model": "asked", "selected": False} for o in others] + if selected is not None: + available.insert(len(available) // 2, {"provider": selected, "model": "asked", "selected": True}) + return {"requested": "asked", "attempt": 1, "summary": f"available={len(available)}, selected={selected}", + "endpoints": {"total": len(available) + 2, "available": available}} + + +# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions and Messages of +# themselves; Responses in the routing facts alone, given when the request asks for them (``ASKED``). +CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}} +CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "h"}, "finish_reason": None}]}, + {"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "i"}, "finish_reason": "length"}]}, + {"id": "c1", "provider": "Prov A", "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1}}) +MESSAGES = {"type": "message", "provider": "Prov B", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}} +MESSAGES_SSE = sse({"type": "message_start", "message": {"type": "message", "provider": "Prov B", "usage": {"input_tokens": 3}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 1}}, + {"type": "message_stop"}) +RESPONSES = {"object": "response", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}], + "usage": {"input_tokens": 3, "output_tokens": 1}} +COMPLETED = {"object": "response", "status": "completed", "output": [], "usage": {"input_tokens": 3, "output_tokens": 1}} +RESPONSES_SSE = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": COMPLETED}) +RESPONSES_ASKED = dict(RESPONSES, openrouter_metadata=routing_facts("Prov C", "Prov D", "Prov E")) +RESPONSES_SSE_ASKED = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov C", "Prov D"))}) +UNASKED = {False: RESPONSES, True: RESPONSES_SSE} +SHAPES = {("/chat/completions", False): (CHAT, "Prov A", "stop"), ("/chat/completions", True): (CHAT_SSE, "Prov A", "length"), + ("/messages", False): (MESSAGES, "Prov B", "end_turn"), ("/messages", True): (MESSAGES_SSE, "Prov B", "max_tokens"), + ("/responses", False): (RESPONSES_ASKED, "Prov C", "incomplete: max_output_tokens"), + ("/responses", True): (RESPONSES_SSE_ASKED, "Prov C", "completed")} + + +def body_for(path: str, stream: bool) -> dict: + """A caller's request of that shape.""" + if path == "/responses": + return {"model": "asked", "input": "hello", "stream": stream} + return {"model": "asked", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5, "stream": stream} + + +def without_new(reply: dict) -> dict: + return {k: v for k, v in reply.items() if k not in NEW} + + +class Upstream(BaseHTTPRequestHandler): + """A provider on the loopback: keeps every request as it came, bytes and headers, and answers it in its shape; + ``Upstream.next`` replaces the next reply (status, content type, body), ``Upstream.always`` every one.""" + seen = [] + next = None + always = None + + def log_message(self, *a): + pass + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + Upstream.seen.append((self.path, {k.lower(): v for k, v in self.headers.items()}, raw)) + req = json.loads(raw) + if Upstream.next is not None or Upstream.always is not None: + (status, kind, body), Upstream.next = Upstream.next or Upstream.always, None + else: + path = "/" + self.path.split("/v1/", 1)[-1] + reply = SHAPES.get((path, bool(req.get("stream"))), (CHAT,))[0] + if path == "/responses" and self.headers.get("X-OpenRouter-Metadata") != "enabled": + reply = UNASKED[bool(req.get("stream"))] + status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ + reply if isinstance(reply, bytes) else json.dumps(reply).encode() + self.send_response(status) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url: str, body: dict, headers=None) -> bytes: + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY, + **(headers or {})}) + with urllib.request.urlopen(req, timeout=10) as r: + return r.read() + + +class WithUpstream(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.tmp = tempfile.mkdtemp() + Upstream.seen, Upstream.next, Upstream.always = [], None, None + self.relays = [] + self.addCleanup(setattr, Upstream, "always", None) + + def tearDown(self): + for r in self.relays: + r.stop() + shutil.rmtree(self.tmp, ignore_errors=True) + + def relay(self, **kw): + r = relay.Relay(self.base, kw.pop("key", "or-test"), os.path.join(self.tmp, f"rec{len(self.relays)}"), **kw).start() + self.relays.append(r) + return r + + def sent(self, i=-1) -> dict: + return json.loads(Upstream.seen[i][2]) + + +class TheRelay(WithUpstream): + def test_the_pin_goes_on_every_request_sent_as_the_pinned_model_and_on_no_other(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0-judge", model="m-judge") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0") + "/messages", body_for("/messages", True)) + self.assertEqual([(json.loads(raw)["model"], json.loads(raw).get("provider")) for _, _, raw in Upstream.seen], + [("m-run", PIN), ("m-judge", None), ("m-run", PIN)]) + self.assertEqual([c["request"].get("routing") for c in r.calls("u0")], [PIN, PIN]) + self.assertNotIn("routing", r.calls("u0-judge")[0]["request"]) + + def test_a_callers_own_routing_is_replaced_and_both_are_recorded(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"sort": "price"})) + self.assertEqual(self.sent()["provider"], PIN) + (c,) = r.calls("u0") + self.assertEqual((c["request"]["routing"], c["request"]["routing_asked"]), (PIN, {"sort": "price"})) + + def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self): + r = self.relay(model="m-run") + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"only": ["prov-z"]})) + self.assertEqual(self.sent()["provider"], {"only": ["prov-z"]}) + (c,) = r.calls("u0") + self.assertEqual(c["request"]["routing"], {"only": ["prov-z"]}) + self.assertNotIn("routing_asked", c["request"]) + + def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): + """An error that echoes the request's credentials, and a routing object holding a token: the record keeps + neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" + Upstream.always = (200, "application/json", json.dumps({"choices": [{"message": {"content": "ok"}}], + "error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + for serving in (None, {"m-run": ["prov-a"]}): + with self.subTest(serving=serving): + r = self.relay(model="m-run", key=KEY, serving=serving) + asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}", KEY: "as a key"}) + post(r.url("u0") + "/chat/completions", asked) + c = r.calls("u0")[-1] + self.assertNotIn(KEY, json.dumps(c)) + self.assertEqual(c["reply"]["error"]["code"], 401) + if serving is None: # sent as the caller sent it: the record alone is cleaned + self.assertEqual(self.sent()["provider"], asked["provider"]) + self.assertEqual(c["request"]["routing"]["token"], "[REDACTED]") + else: + self.assertEqual(c["request"]["routing_asked"]["token"], "[REDACTED]") + + def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + for (path, stream), (_, who, end) in SHAPES.items(): + with self.subTest(path=path, stream=stream): + post(r.url("u0") + path, body_for(path, stream)) + self.assertEqual(self.sent()["provider"], {"order": ["prov-a"], "allow_fallbacks": False}) + reply = r.calls("u0")[-1]["reply"] + self.assertEqual((reply.get("served_by"), reply.get("end"), reply.get("error")), (who, end, None)) + # what was read before is read as before, beside the new facts + self.assertEqual(without_new(reply), relay.read_reply(path, SHAPES[(path, stream)][0] if stream + else json.dumps(SHAPES[(path, stream)][0]).encode(), stream)) + + def test_several_choices_keep_their_endings_and_errors_by_index(self): + body = {"choices": [{"index": 0, "message": {"content": "ok"}, "finish_reason": "stop"}, + {"index": 1, "message": {}, "finish_reason": "error", "error": {"code": 429, "message": "limited"}}]} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(body).encode(), False), + {"end": {"0": "stop", "1": "error"}, "error": {"code": 429, "message": "limited", "choice": 1}}) + stream = sse({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}, {"index": 1, "delta": {}, "finish_reason": None}]}, + {"choices": [{"index": 1, "delta": {}, "finish_reason": "stop"}]}) + self.assertEqual(relay.reply_facts("/chat/completions", stream, True)["end"], {"0": "length", "1": "stop"}) + + def test_an_error_beside_a_whole_reply_is_read_in_every_shape(self): + error = {"code": 502, "message": "failed"} + for path, body in (("/messages", {"type": "message", "content": [], "error": error}), + ("/embeddings", {"object": "list", "data": [], "error": error}), + ("/chat/completions", {"choices": [{"message": {"content": "ok"}}], "error": error}), + ("/responses", {"object": "response", "status": "failed", "output": [], "error": error})): + with self.subTest(path=path): + self.assertEqual(relay.reply_facts(path, json.dumps(body).encode(), False)["error"], error) + + def test_a_completed_responses_events_routing_facts_name_who_served_it(self): + body = sse({"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov A", "Prov X"))}) + self.assertEqual(relay.reply_facts("/responses", body, True), {"served_by": "Prov A", "end": "completed"}) + whole = json.dumps(dict(RESPONSES, openrouter_metadata=routing_facts("Prov A", "Prov X", "Prov Y"))).encode() + self.assertEqual(relay.reply_facts("/responses", whole, False)["served_by"], "Prov A") + + def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): + odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"endpoints": {"available": 1}}}, + {"type": "message", "content": [], "provider": 7, "error": "a string", "stop_reason": ["x"]}, + {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}, + {"choices": [], "openrouter_metadata": {"endpoints": [1]}}, + {"choices": [], "openrouter_metadata": {"endpoints": {"available": [7, {"provider": 7, "selected": True}, + {"provider": "Prov A", "selected": "true"}]}}}] + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + for body in odd: + with self.subTest(body=body): + raw = json.dumps(body).encode() + Upstream.next = (200, "application/json", raw) + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)), raw) + self.assertEqual({k: v for k, v in r.calls("u0")[-1]["reply"].items() if k in NEW}, {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"[1, 2]", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) + + def test_a_failed_responses_stream_is_read_beside_its_facts(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage are read from its + ``response.failed`` event, its error beside them.""" + body = sse({"type": "response.output_text.delta", "delta": "part"}, + {"type": "response.failed", "response": {"object": "response", "status": "failed", + "error": {"code": "server_error", "message": "failed"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], + "usage": {"input_tokens": 5, "output_tokens": 2}}}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "x", "tool_calls": [], "usage": relay._usage({"input_tokens": 5, "output_tokens": 2})}) + self.assertEqual(relay.reply_facts("/responses", body, True), + {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) + + def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): + """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own + is added, and a unit's failure reads it as before (a 200 is no failure).""" + r = self.relay(model="m-run") + errors = [(sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, + "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", + {"code": "server_error", "message": "upstream ended"}), + (sse({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), "text/event-stream", + {"type": "overloaded_error", "message": "Overloaded"})] + for body, kind, said in errors: + with self.subTest(said=said): + Upstream.next = (200, kind, body) + streamed = kind == "text/event-stream" + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", streamed)), body) + c = r.calls("u0")[-1] + self.assertEqual((c["status"], c["reply"]["error"]), (200, said)) + self.assertNotIn("error", c) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", body, streamed)) + self.assertIsNone(relay.failure(r.calls("u0"))) + + def test_the_routing_facts_are_asked_of_openrouter_alone_and_always_as_enabled(self): + meta = json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, + openrouter_metadata=routing_facts("Prov A", "Prov X"))).encode() + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + Upstream.next = (200, "application/json", meta) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False), {"X-OpenRouter-Metadata": "disabled"}) + got = [v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"] + self.assertEqual(got, ["enabled"]) + self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") + for stream in (False, True): # a Responses request: asked too + post(r.url("u0") + "/responses", body_for("/responses", stream), {"X-OpenRouter-Metadata": "disabled"}) + self.assertEqual([v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"], ["enabled"]) + post(r.url("u0") + "/messages", body_for("/messages", False)) # a Messages reply names its provider: nothing asked + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + for kw in ({"serving": {"m-run": ["prov-a"]}}, {"facts": True}): # another provider's address; no pin + with self.subTest(kw=kw): + other = self.relay(model="m-run", **kw) + post(other.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + self.assertTrue(relay.Relay("https://openrouter.ai/api/v1", "k", os.path.join(self.tmp, "or")).facts) + self.assertFalse(relay.Relay("https://api.openai.com/v1", "k", os.path.join(self.tmp, "oa")).facts) + + def test_without_a_pin_every_request_goes_on_byte_for_byte_as_before(self): + """What a relay sent before pins existed: the caller's body with the model set (and a streamed Chat + Completions request's usage flag), serialised once, and the caller's headers but its credentials. The + three shapes, streamed or not, with and without a routing field of the caller's, as the run's model, a + side model and a judge's route's model; with no pin, or a pin on another model.""" + for serving in (None, {"m-other": ["prov-a"]}): + r = self.relay(model="m-run", side_models=("m-side",), serving=serving) + for (path, stream) in SHAPES: + for extra in ({}, {"provider": {"order": ["prov-z"]}}): + for url, asked, model in ((r.url("u0"), "asked", "m-run"), (r.url("u0"), "m-side", "m-side"), + (r.url("u0-judge", model="m-judge"), "asked", "m-judge")): + with self.subTest(serving=serving, path=path, stream=stream, extra=extra, model=model): + req = dict(body_for(path, stream), model=asked, **extra) + post(url + path, req, {"X-Caller": "kept"}) + expected = dict(req, model=model) + if stream and path == "/chat/completions": + expected.setdefault("stream_options", {})["include_usage"] = True + sent_path, headers, raw = Upstream.seen[-1] + self.assertEqual(raw, json.dumps(expected).encode()) + self.assertEqual(sent_path, "/v1" + path) + self.assertEqual({k: v for k, v in headers.items() if k not in ("host", "content-length")}, + {"content-type": "application/json", "x-caller": "kept", "accept": "*/*", + "accept-encoding": "identity", "authorization": "Bearer or-test", + "user-agent": headers["user-agent"]}) + + +class TheSidecar(WithUpstream): + def setUp(self): + super().setUp() + self.enterContext(mock.patch.object(container, "_security", "")) # a default rootful engine, never asked + + def test_the_containers_relay_is_given_the_pin_on_its_command_line(self): + argv = container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", base="http://x/v1", + model="m-run", routes={"u0-judge": "m-judge"}, + serving={"m-run": ["prov-a", "prov-b/fp8"], "m-judge": ["prov-c"]}) + got = [argv[i + 1] for i, a in enumerate(argv) if a == "--served-by"] + self.assertEqual(got, ["m-judge=prov-c", "m-run=prov-a,prov-b/fp8"]) + self.assertNotIn("--served-by", container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", + base="http://x/v1", model="m-run", routes={})) + + def test_the_unit_hands_its_pin_to_the_sidecars_command(self): + unit = container.Unit("job", "u0", os.path.join(self.tmp, "rec"), image="img", app=APP, base=self.base, key="or-test", + model="m-run", serving={"m-run": ["prov-a"]}) + ran = [] + + def docker(args, **kw): + ran.append(args) + if "run" in args[:1]: + raise container.Unavailable("no engine in this test") + return subprocess.CompletedProcess(args, 0, "", "") + with mock.patch.object(container, "_docker", docker), self.assertRaises(container.Unavailable): + unit.start() + (started,) = [a for a in ran if a[:1] == ["run"]] + self.assertIn("m-run=prov-a", started[started.index("--served-by") + 1:]) + + def test_the_relays_own_command_refuses_a_pin_it_cannot_take(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + for given in (["m=bad slug"], ["m=prov-a,prov-a"], ["m="], ["m=prov-a", "m=prov-b"], ["prov-a"]): + with self.subTest(given=given): + # a process of its own, bounded: a relay that took the pin would serve until stopped + got = subprocess.run([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", + "--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, + *[w for g in given for w in ("--served-by", g)]], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), capture_output=True, text=True, timeout=20) + self.assertEqual(got.returncode, 2, got.stdout) + self.assertIn("error", json.loads(got.stdout.splitlines()[0])) + + def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + p = subprocess.Popen([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", "--base", self.base, + "--record", os.path.join(self.tmp, "rec"), "--key-file", key, "--model", "m-run", + "--served-by", "m-run=prov-a,prov-b/fp8", "--routing"], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) + try: + ready, _, _ = select.select([p.stdout], [], [], 20) # its one line, or no more than 20 s + self.assertTrue(ready, "the relay did not say it listens within 20 s") + url = json.loads(p.stdout.readline())["listening"] + post(url + "/u0/v1/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(self.sent()["provider"], PIN) + finally: + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + p.wait(10) + p.stdout.close() + + +class TheRun(WithUpstream): + def setUp(self): + super().setUp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = definitions.get("benchmark", "fakebench", ROOT) + self.a = definitions.get("agent", "fake", ROOT) + + def tearDown(self): + install.HOME = self._home + super().tearDown() + + def make(self, **kw): + kw.setdefault("other", self.a) + return run.Run(kw.pop("bench", self.b), kw.pop("other"), model=kw.pop("model", "m-1"), provider=kw.pop("provider", "openrouter"), + env=kw.pop("env", {"OPENROUTER_API_KEY": "or-test"}), unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, **kw) + + def test_the_option_reads_model_and_slugs_and_refuses_what_it_cannot_take(self): + self.assertEqual(run.served_by(["prov-a", "m-judge=prov-b/fp8, prov-c"], "m-run"), + {"m-run": ["prov-a"], "m-judge": ["prov-b/fp8", "prov-c"]}) + for given, said in ((["m-run=prov-a,"], "not a serving provider's slug"), (["m-run=a b"], "not a serving provider's slug"), + (["prov-a,prov-a"], "named twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), + (["m-run="], "not a serving provider's slug")): + with self.subTest(given=given), self.assertRaisesRegex(run.Refused, said): + run.served_by(given, "m-run") + with self.assertRaisesRegex(run.Refused, "MODEL=SLUG"): + run.served_by(["prov-a"], "") + + def test_a_run_refuses_a_pin_it_cannot_take_however_it_is_given(self): + for pin in ({"m-1": []}, {"m-1": ["bad slug"]}, {"m-1": ["prov-a", "prov-a"]}, {"": ["prov-a"]}, {"m-1": "prov-a"}): + with self.subTest(pin=pin), self.assertRaisesRegex(run.Refused, "--served-by"): + self.make(served_by=pin) + + def test_a_provider_that_routes_no_model_to_several_is_refused_before_any_unit(self): + for provider, env in (("openai", {"OPENAI_API_KEY": "sk-x"}), ("anthropic", {"ANTHROPIC_API_KEY": "x"}), + ("custom", {"MODEL_API_URL": self.base})): + with self.subTest(provider=provider), self.assertRaisesRegex(run.Refused, "no serving providers to choose among"): + self.make(provider=provider, env=env, served_by={"m-1": ["prov-a"]}) + + def test_a_model_the_run_does_not_call_is_refused_before_any_unit(self): + with self.assertRaisesRegex(run.Refused, "names m-other, which this run does not call: it calls m-1"): + self.make(served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + nop = definitions.control(dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), "nop") + with self.assertRaisesRegex(run.Refused, "names m-1, which this run does not call: it calls no model"): + self.make(other=nop, bench=dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), + served_by={"m-1": ["prov-a"]}) + + def test_the_command_line_refuses_each_with_exit_2(self): + for patch in (mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_other", return_value=self.a), + mock.patch.object(cli, "_exit_on_term"), mock.patch.object(cli.settings, "load"), + mock.patch.object(progress, "ECHO", None)): + patch.start() + self.addCleanup(patch.stop) + for extra, said in ((["--provider", "openai", "--served-by", "prov-a"], "no serving providers"), + (["--served-by", "m-2=prov-a"], "does not call"), + (["--served-by", "m-1=bad slug"], "not a serving provider's slug"), + (["--served-by", "m-1="], "not a serving provider's slug"), + (["--served-by", "prov-a,prov-a"], "named twice"), + (["--served-by", "prov-a", "--served-by", "m-1=prov-b"], "m-1 twice")): + with self.subTest(extra=extra), mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-x", "OPENROUTER_API_KEY": "or-test"}): + out = io.StringIO() + with redirect_stdout(out): + code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--provider", "openrouter", + "--jobs", os.path.join(self.tmp, "jobs"), *extra]) + self.assertEqual(code, 2) + self.assertTrue(out.getvalue().strip().splitlines()[-1].startswith("refused:"), out.getvalue()) + self.assertIn(said, out.getvalue()) + + def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + with mock.patch.dict(run.PROVIDERS, {"openrouter": dict(run.PROVIDERS["openrouter"], base=self.base)}): + job_dir = run.run(b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding", + served_by={"m-1": ["prov-a"]}) + self.assertTrue(Upstream.seen) + self.assertTrue(all(json.loads(raw).get("provider") == {"order": ["prov-a"], "allow_fallbacks": False} + for _, _, raw in Upstream.seen)) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + trials = {} + for d in os.listdir(job_dir): + if "__u" in d: + with open(os.path.join(job_dir, d, "result.json")) as f: + trials[d.split("__")[0]] = json.load(f)["standard"] + n = len(Upstream.seen) + self.assertEqual(trials["model"]["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + self.assertEqual(trials["good"]["served"], {"calls": {}, "unnamed": 0, "held": {"m-1": None}}) # it called no model + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + self.assertTrue(all(c["request"]["routing"]["order"] == ["prov-a"] and c["reply"]["served_by"] == "Prov A" for c in calls)) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + + def test_a_run_without_the_option_sends_records_and_ends_as_before(self): + """Every request as before, the record's earlier fields as they were read, and the same verdicts, even + when the provider answers a 200 that holds an error: the relay reads that error beside the reply and the + unit ends as it did.""" + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + reply = dict({k: v for k, v in CHAT.items() if k != "provider"}, error={"code": 502, "message": "Provider returned error"}) + Upstream.always = (200, "application/json", json.dumps(reply).encode()) + job_dir = run.run(b, self.a, model="m-1", provider="custom", env={"MODEL_API_URL": self.base, "MODEL_API_KEY": "k"}, + unsandboxed=True, jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding") + self.assertTrue(Upstream.seen) + for _, headers, raw in Upstream.seen: + self.assertNotIn("provider", json.loads(raw)) + self.assertNotIn("x-openrouter-metadata", headers) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("served_by", json.load(f)["standard"]) + with open(os.path.join(job_dir, "model__u0002", "result.json")) as f: + res = json.load(f) + self.assertNotIn("exception_info", res) # a 200 is no failure: the unit's verdict stands + self.assertEqual(res["verifier_result"]["rewards"]["reward"], 0.0) + self.assertNotIn("served", res["standard"]) # no call answered, no pin: nothing to say + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + for c in calls: + self.assertEqual((c["status"], "error" in c), (200, False)) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", json.dumps(reply).encode(), False)) + self.assertEqual(c["reply"]["error"], {"code": 502, "message": "Provider returned error"}) + self.assertNotIn("routing", c["request"]) + + def test_the_summary_counts_every_route_of_the_unit(self): + """A judge on the pinned model, served by another provider than the agent's: the pin does not hold.""" + r = self.make(judge_model="m-1", served_by={"m-1": ["prov-a"]}) + call = lambda who: {"model": "m-1", "status": 200, "reply": {"served_by": who}} + routes = {"u0000": [call("Prov A")], "u0000-judge": [call("Prov Z")], "u0000-simulator": [call("Prov A")]} + rr = mock.Mock(mcp_listeners=[], unit_image="") + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 1} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit", "note_refusal"): + stack.enter_context(mock.patch.object(r, name)) + stack.enter_context(mock.patch.object(r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = r.unit(0, "good") + self.assertEqual(res["standard"]["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + self.assertEqual(res["agent_result"]["n_model_calls"], 1) # the tokens and cost: the agent's calls, as before + + def test_the_benchmarks_own_relay_counts_in_the_job(self): + r = self.make(served_by={"m-1": ["prov-a"]}) + folder = os.path.join(r.job_dir, ".work", "relay-benchmark") + os.makedirs(folder) + with open(os.path.join(folder, "embed-1.jsonl"), "w") as f: + f.write(json.dumps({"model": "m-1", "status": 200, "reply": {"served_by": "Prov Z"}}) + "\n") + bench = job.served(r.benchmark_calls(), r.serving) + job.write_result(r.job_dir, r.name, 0, 1, [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 0, "held": {"m-1": True}}}}], + 1, served=bench) + with open(os.path.join(r.job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + def test_the_containers_unit_is_given_the_runs_pin(self): + with mock.patch.object(container, "unusable", return_value=None): + r = run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, backend="container", + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + r.image, r.setup_driver, r.licences = "img", None, run.Licences(self.b, given={}) + trial = os.path.join(self.tmp, "trial") + with mock.patch.object(container, "Unit") as unit, mock.patch.object(run.Run, "unit_image", return_value="img"): + r.unit_relay("u0000", trial, {}) + self.assertEqual(unit.call_args.kwargs["serving"], {"m-1": ["prov-a"]}) + + +class TheRecord(unittest.TestCase): + def call(self, model, who=None, status=200, error=None): + reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} + return {"model": model, "status": status, "reply": reply} + + def test_who_served_a_units_calls_and_whether_the_pin_held(self): + pin = {"m-1": ["google-ai-studio", "prov-b/fp8"], "m-judge": ["prov-c"]} + self.assertEqual(job.served([self.call("m-1", "Google AI Studio"), self.call("m-1", "Prov B"), self.call("m-1", status=429)], pin), + {"calls": {"Google AI Studio": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": True, "m-judge": None}}) + self.assertEqual(job.served([self.call("m-1", "Prov Z"), self.call("m-1")], pin)["held"]["m-1"], "unknown") + self.assertEqual(job.served([self.call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered + self.assertEqual(job.served([self.call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) + self.assertIsNone(job.served([self.call("m-1", status=429)], None)) + + def test_a_name_reads_as_a_slug_by_its_case_and_spaces_alone(self): + pin = {"m-1": ["prov-a"]} + for name, held in (("Prov A", True), ("prov-a", True), ("PROV A", True), ("Prov/A", "unknown"), ("Prov_A", "unknown"), + ("Prov.A", "unknown"), ("Prov Ab", "unknown")): + with self.subTest(name=name): + self.assertEqual(job.served([self.call("m-1", name)], pin)["held"]["m-1"], held) + + def test_answered_calls_that_name_no_provider_are_counted(self): + known, unknown = job.served([self.call("m-1", "Prov A")]), job.served([self.call("m-1")]) + self.assertEqual(unknown, {"calls": {}, "unnamed": 1}) + self.assertEqual(job.served_total([{"standard": {"served": known}}, {"standard": {"served": unknown}}, {"standard": {}}]), + {"calls": {"Prov A": 1}, "unnamed": 1}) + parts = [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 1, "held": {"m-1": True}}}}, + {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}] + self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) + + +class TheReview(unittest.TestCase): + """What a review of the pin found, each a rule of its own.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_the_selected_endpoint_of_the_replys_end_is_recorded_never_another(self): + """The routing facts list the endpoints available: the one marked selected served the reply, never the + first listed nor one an earlier event's facts marked.""" + events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov B", "Prov A", "Prov C"))}] + facts = relay.reply_facts("/responses", sse(*events), True) + held = job.served([{"status": 200, "model": "m", "reply": facts}], {"m": ["prov-a"]})["held"]["m"] + self.assertEqual((facts["served_by"], held), ("Prov B", "unknown")) + body = dict(RESPONSES, openrouter_metadata=routing_facts("Prov B", "Prov A")) + self.assertEqual(relay.reply_facts("/responses", json.dumps(body).encode(), False)["served_by"], "Prov B") + + def test_facts_that_mark_no_single_named_endpoint_take_no_earlier_name(self): + both = routing_facts("Prov A", "Prov X") + for e in both["endpoints"]["available"]: + e["selected"] = True + for meta in (routing_facts(None, "Prov A", "Prov X"), routing_facts(7, "Prov A"), both): + with self.subTest(meta=meta): + body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": meta} + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) + stream = sse({"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=meta)}) + self.assertNotIn("served_by", relay.reply_facts("/responses", stream, True)) + chunks = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "openrouter_metadata": meta}) + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", chunks, True)) + + def test_a_pinned_responses_call_is_recorded_with_its_provider_and_the_pin_holds(self): + """A Responses reply names its provider in the routing facts alone: the relay asks for them on a pinned + request to OpenRouter, records the endpoint they mark, and the pin reads held, streamed or not. Without + them the reply names no provider and the pin reads unknown.""" + up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=up.serve_forever, daemon=True).start() + self.addCleanup(up.server_close) + self.addCleanup(up.shutdown) + Upstream.seen, Upstream.next, Upstream.always = [], None, None + base, pin = f"http://127.0.0.1:{up.server_address[1]}/v1", {"m-run": ["prov-c"]} + for facts, who, held in ((True, "Prov C", True), (False, None, "unknown")): + r = relay.Relay(base, "or-test", os.path.join(self.tmp, f"rec-{facts}"), model="m-run", serving=pin, facts=facts).start() + self.addCleanup(r.stop) + for stream in (False, True): + with self.subTest(facts=facts, stream=stream): + post(r.url("u0") + "/responses", body_for("/responses", stream)) + self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled" if facts else None) + self.assertEqual(json.loads(Upstream.seen[-1][2])["provider"], {"order": ["prov-c"], "allow_fallbacks": False}) + self.assertEqual(r.calls("u0")[-1]["reply"].get("served_by"), who) + self.assertEqual(job.served(r.calls("u0"), pin)["held"], {"m-run": held}) + + def test_an_error_keeps_its_stated_fields_alone(self): + nested = relay.reply_facts("/chat/completions", json.dumps({"error": {"api_key": ["review-only-credential"]}}).encode(), False) + self.assertNotIn("review-only-credential", json.dumps(nested)) + self.assertEqual(relay.reply_facts("/messages", b'{"type":"message","content":[],"error":{"code":[],"type":{},"message":17}}', False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b'{"error":{"code":502,"message":"m","detail":{"key":"x"}}}', False), + {"error": {"code": 502, "message": "m"}}) + + def test_of_several_choices_errors_the_top_level_one_else_the_first_is_kept(self): + two = [{"index": 0, "finish_reason": "error", "error": {"code": 429, "message": "rate limited"}}, + {"index": 1, "finish_reason": "error", "error": {"code": 503, "message": "unavailable"}}] + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps({"choices": two}).encode(), False)["error"], + {"code": 429, "message": "rate limited", "choice": 0}) + top = {"choices": two, "error": {"code": 500, "message": "whole"}} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(top).encode(), False)["error"], {"code": 500, "message": "whole"}) + + def test_a_merged_job_keeps_each_shards_whole_summary(self): + """A shard's summary holds its benchmark relay's calls too: a merge made of its trials alone would drop + them, and an ``unknown`` would read true.""" + pin = {"m": ["prov-a"]} + trial = {"standard": {"served": job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov A"}}], pin), "shard_job": "s1"}} + bench = job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov B"}}], pin) + shard = {"name": "s1", "config": {"standard": {"served_by": pin}}, "result": {"served": job.served_total([trial], bench)}} + self.assertEqual(merge._served([shard], [trial]), shard["result"]["served"]) + old = {"name": "s2", "config": {"standard": {"served_by": pin}}, "result": {}} # a shard written before: its trials + self.assertEqual(merge._served([old], [dict(trial, standard=dict(trial["standard"], shard_job="s2"))])["held"], {"m": True}) + + def test_a_control_is_refused_a_pin_on_its_simulated_users_model(self): + b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing"}}, + simulator={"model": "sim"}) + with self.assertRaisesRegex(run.Refused, "names sim, which this run does not call"): + run.Run(b, definitions.control(b, "nop"), model="sim", provider="openrouter", env={}, + jobs_root=self.tmp, say=lambda _: None, served_by={"sim": ["prov-a"]}) + + def test_the_relays_own_command_refuses_another_provider_and_a_model_it_does_not_send(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("review-test-key") + for base, model, extra in (("https://api.openai.com/v1", "m", []), ("https://openrouter.ai/api/v1", "unused", []), + ("http://127.0.0.1:9/v1", "m", [])): + with self.subTest(base=base, model=model): + out = io.StringIO() + with mock.patch.object(relay, "Relay") as fake, redirect_stdout(out): + code = relay.serve(["--base", base, "--model", "m", "--key-file", key, "--record", self.tmp, + "--served-by", model + "=prov-a", *extra]) + self.assertEqual(code, 2) + self.assertIn("--served-by", json.loads(out.getvalue())["error"]) + fake.assert_not_called() + + +class TheMerge(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing said"}}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def shards(self, *pins): + jobs = [run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=os.path.join(self.tmp, "jobs"), + env={}, say=lambda s: None, shard=(i, len(pins)), launch="L", name=f"run-shard-{i}of{len(pins)}") + for i in range(1, len(pins) + 1)] + for job_dir, pin in zip(jobs, pins): + path = os.path.join(job_dir, "config.json") + with open(path) as f: + cfg = json.load(f) + cfg["standard"]["served_by"] = pin + with open(path, "w") as f: + json.dump(cfg, f) + return jobs + + def test_shards_of_different_pins_are_refused(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + self.assertEqual(merge.check(jobs, force=True)["differ"][0].split(":")[0], "serving providers") + jobs = self.shards({"m-1": ["prov-a", "prov-b"]}, {"m-1": ["prov-b", "prov-a"]}) # the order is the pin's + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + + def test_equal_pins_merge_whatever_the_order_of_their_models(self): + jobs = self.shards({"a": ["prov-a"], "b": ["prov-b"]}, {"b": ["prov-b"], "a": ["prov-a"]}) + self.assertEqual(merge.check(jobs)["differ"], []) + + def test_a_forced_merge_of_different_pins_holds_no_pin(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + for job_dir, who in zip(jobs, ("Prov A", "Prov B")): + trial = sorted(d for d in os.listdir(job_dir) if "__u" in d)[0] + path = os.path.join(job_dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["served"] = {"calls": {who: 1}, "unnamed": 0, "held": {"m-1": True}} + with open(path, "w") as f: + json.dump(res, f) + out = os.path.join(self.tmp, "merged") + merge.merge(jobs, out, force=True) + with open(os.path.join(out, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + with open(os.path.join(out, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_served_by_constraints.py b/app/tests/test_standard_served_by_constraints.py new file mode 100644 index 0000000..b606c49 --- /dev/null +++ b/app/tests/test_standard_served_by_constraints.py @@ -0,0 +1,172 @@ +"""Provider pins and machine constraints on the same run and recipe. + +These intersections need both features: neither side's tests exercise them +together. The fixture driver scores locally; no endpoint or engine is used. +""" +from __future__ import annotations + +import io +import json +import os +from pathlib import Path +import shlex +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from nullius import constraints, progress +from nullius.standard import cli, container, definitions, install, job, merge, recipe, run +from tests import test_standard_recipe_served_by as recipes + + +class Together(unittest.TestCase): + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(mock.patch.object(install, "HOME", str(self.tmp / "install"))) + self.addCleanup(container.cap, {}) + self.b = recipes.judged() + self.a = definitions.get("agent", "fake", recipes.ROOT) + self.pin = {"m-judge": ["prov-a"]} + + def make(self, **kw): + values = dict(model="m", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, + judge_model="m-judge", served_by=self.pin, unsandboxed=True, + jobs_root=str(self.tmp), say=lambda _: None) + values.update(kw) + return run.Run(self.b, self.a, **values) + + def test_the_cli_records_units_constraints_and_pins_together(self): + units = self.tmp / "units.txt" + units.write_text("good\n") + self.enterContext(mock.patch.object(cli, "_bench", return_value=self.b)) + self.enterContext(mock.patch.object(cli, "_exit_on_term")) + self.enterContext(mock.patch.object(cli.settings, "load")) + self.enterContext(mock.patch.object(progress, "ECHO", None)) + self.enterContext(mock.patch.object(run.relay.Relay, "start", lambda r: r)) + self.enterContext(mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9")) + self.enterContext(mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"})) + seen = [] + proc = run.jsonl.Proc + + def driver(cmd, cwd, env, log, **kw): + seen.append(env) + return proc(cmd, cwd, env, log, **kw) + + self.enterContext(mock.patch.object(run.jsonl, "Proc", driver)) + for i, extra in enumerate(([], ["--from-run"])): + out = io.StringIO() + with self.subTest(extra=extra), redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--jobs", str(self.tmp), "--name", f"job-{i}", + "--units", str(units), "-j", "4", "--constraint", "concurrency=1", + "--constraint", "gpu_ids=", "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", *extra]) + self.assertEqual(code, 0, out.getvalue()) + path = Path(out.getvalue().strip().splitlines()[-1]) + cfg = json.loads((path / "config.json").read_text()) + self.assertEqual(cfg["standard"]["units_file"], run.units_file(str(units))) + self.assertEqual(cfg["standard"]["constraints"], {"concurrency": 1, "gpu_ids": []}) + self.assertEqual(cfg["standard"]["served_by"], self.pin) + self.assertEqual(cfg["n_concurrent_trials"], 1) + self.assertEqual(sorted(p.name for p in path.iterdir() if "__u" in p.name), ["good__u0000"]) + self.assertEqual(json.loads((path / "result.json").read_text())["served"]["held"], {"m-judge": None}) + self.assertTrue(seen) + self.assertTrue(all(env["CUDA_VISIBLE_DEVICES"] == "" for env in seen)) + + def test_constraint_and_host_refusals_precede_pin_and_endpoint_checks(self): + with mock.patch.object(run, "provider_endpoint") as endpoint: + with self.assertRaisesRegex(run.Refused, "constraints:.*at least 1"): + self.make(constraints={"concurrency": 0}, served_by={"stray": ["prov-a"]}) + self.b = dict(self.b, host={"gpu": {"check": ["nvidia-smi"]}}, + options={"compute": {"default": "gpu", "values": ["gpu"], "needs": {"gpu": {"host": ["gpu"]}}}}) + with self.assertRaisesRegex(run.Refused, "allows Nullius no GPU"): + self.make(constraints={"gpu_ids": []}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + with self.assertRaisesRegex(run.Refused, "--served-by.*does not call"): + self.make(constraints={"gpu_ids": [0]}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + endpoint.assert_not_called() + out = io.StringIO() + with mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_exit_on_term"), \ + mock.patch.object(cli.settings, "load"), mock.patch.object(run, "run_and_report") as start, redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--constraint", "concurrency=0", "--served-by", "bad slug"]) + self.assertEqual(code, 2) + self.assertIn("refused: constraints:", out.getvalue()) + start.assert_not_called() + + def test_a_valid_pin_still_requires_enforced_machine_caps(self): + r = self.make(constraints={"cpus": 1}) + with mock.patch.object(constraints, "scope_check", return_value={"why": "unavailable", "enforced": [], "refused": {}}), \ + mock.patch.object(install, "packages") as setup, self.assertRaisesRegex(run.Refused, "systemd does not enforce"): + r.ready() + setup.assert_not_called() + + def test_the_container_relay_takes_both_caps_and_pins(self): + self.enterContext(mock.patch.object(container, "_security", "")) + with mock.patch.object(container, "unusable", return_value=None): + r = self.make(backend="container", constraints={"cpus": 1, "memory_mb": 512}) + argv = container.relay_argv("img", name="relay", job="job", app="/app", record="/rec", key_file="/key", + base="https://openrouter.ai/api/v1", model="m", routes={"u0-judge": "m-judge"}, serving=r.serving) + self.assertEqual(argv[argv.index("--cpus") + 1], "1") + self.assertEqual(argv[argv.index("--memory") + 1], "512m") + self.assertEqual(argv[argv.index("--served-by") + 1], "m-judge=prov-a") + with mock.patch.object(r, "scope_ready", return_value={}), \ + mock.patch.object(container, "base_tag", return_value="img"), \ + mock.patch.object(container, "present", return_value=True), \ + mock.patch.object(container, "base_image", return_value="img"), \ + mock.patch.object(container, "unenforced", return_value={"cpus": "max 100000"}), \ + self.assertRaisesRegex(run.Refused, "engine does not apply"): + r.ready() + + def test_each_compiled_shard_has_its_constraints_and_each_called_models_pin(self): + given = {"local": {"concurrency": 1}, "127.0.0.1": {"cpus": 2, "gpu_ids": []}} + with mock.patch.dict(os.environ, {constraints.SETTING: json.dumps(given)}): + out = recipe.compile(recipes.a_recipe(where=list(given), others=["fake", "nop"], + model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), recipes.defs(self.b)) + self.assertEqual(out["errors"], []) + self.assertEqual(out["served_models"]["model"], "openai/gpt-4o-mini") + for part in out["shards"]: + for line in part["command"].splitlines(): + words = shlex.split(line) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--constraint"], + constraints.cli_words(given[part["where"]])[1::2]) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--served-by"], + ["openai/gpt-4o-mini=prov-a"]) + self.assertLess(words.index("--constraint"), words.index("--served-by")) + + def test_builder_keeps_both_validation_errors_in_main_first_order(self): + with mock.patch.dict(os.environ, {constraints.SETTING: '{"local": {"cpus": 0}}'}): + out = recipe.compile(recipes.a_recipe(model="openai/gpt-4o-mini", + served_by={"model": ["prov-a"], "judge": ["prov-b"]}), recipes.defs(self.b)) + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"], recipe.STEP["model"]]) + self.assertIn("HOST_CONSTRAINTS", out["errors"][0]["message"]) + self.assertIn("pins openai/gpt-4o-mini twice", out["errors"][1]["message"]) + self.assertEqual((out["script"], out["parts"]), ("", [])) + + def test_merge_allows_machine_limits_to_differ_but_still_checks_pins(self): + paths = [] + for i in (1, 2): + path = self.tmp / f"shard-{i}" + path.mkdir() + job.write_config(str(path), job=path.name, bench=self.b, other=self.a, model="m", provider="openrouter", + dataset="all", units=[], concurrency=i, granted=True, launch="launch", + shard={"index": i, "of": 2, "units_total": 0, "planned": []}, + constraints={"concurrency": i}, served_by=self.pin) + (path / "result.json").write_text(json.dumps({"served": {"calls": {"Prov A": 1}, "unnamed": 0, + "held": {"m-judge": True}}})) + paths.append(str(path)) + self.assertEqual(merge.check(paths)["differ"], []) + merge.merge(paths, str(self.tmp / "joined")) + cfg = json.loads((self.tmp / "joined/config.json").read_text())["standard"] + self.assertEqual((cfg["constraints"], cfg["served_by"]), ({"concurrency": 1}, self.pin)) + self.assertEqual(json.loads((self.tmp / "joined/result.json").read_text())["served"]["held"], {"m-judge": True}) + path = Path(paths[1]) / "config.json" + cfg = json.loads(path.read_text()) + cfg["standard"]["served_by"] = {"m-judge": ["prov-b"]} + path.write_text(json.dumps(cfg)) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(paths) + merge.merge(paths, str(self.tmp / "forced"), force=True) + self.assertEqual(json.loads((self.tmp / "forced/result.json").read_text())["served"]["held"], {"m-judge": "unknown"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index 7d84a04..751625c 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -15,7 +15,7 @@ from types import SimpleNamespace from nullius.runs import launches, parse -from nullius.standard import job, merge, shard +from nullius.standard import job, merge, prove, shard BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} @@ -61,14 +61,15 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local"): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", bench_options=None): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) os.makedirs(d) planned = list(units) if planned is None else planned job.write_config(d, job=name, bench=bench, other=other, model=model, provider="openai", dataset="all", units=planned, - concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend) + concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend, + bench_options=bench_options) results = [] for k, u in enumerate(units): place = int(u.split("-")[1]) @@ -80,7 +81,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": 1.0 if place % 2 else 0.0}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) + job.write_result(d, name, t0, t0 + 10 * (min(index, 9) + 1), results, len(planned), budget=budget) return d @@ -199,6 +200,36 @@ def test_units_no_shard_ran_are_said(self): j = parse.parse_job(got["out"]) self.assertEqual((j.n_total_trials, len(j.trials)), (len(self.split[1]) + len(planned), len(self.split[1]) + len(planned) - 1)) + def test_a_launch_of_one_unit_per_shard_merges_and_says_only_the_units_it_lacks(self): + n = 2 ** 64 + units = [f"t-{i}" for i in range(10)] + dirs = [write_shard(self.tmp, f"one-{u}", [u], shard.of(u, n), n) for u in units[:-1]] + got = merge.merge(dirs, os.path.join(self.tmp, "out")) + self.assertEqual((got["trials"], got["notes"]), (9, ["1 of the dataset's 10 units are not in the merge"])) + + def test_shards_run_with_different_benchmark_options_keep_each_its_own(self): + same = {"websites": "none", "gitlab": "none"} + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, bench_options=same), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example")), + write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)] + std = read(os.path.join(merge.merge(dirs, os.path.join(self.tmp, "out"))["out"], "config.json"))["standard"] + self.assertNotIn("benchmark_options", std) + self.assertEqual([m["benchmark_options"]["gitlab"] for m in std["merged_from"]], ["none", "gitlab.example", "none"]) + alike = [write_shard(self.tmp, f"a{i}", self.split[i], i, 3, bench_options=same) for i in (1, 2, 3)] + std = read(os.path.join(merge.merge(alike, os.path.join(self.tmp, "alike"))["out"], "config.json"))["standard"] + self.assertEqual(std["benchmark_options"], same) + self.assertNotIn("benchmark_options", std["merged_from"][0]) + + def test_a_merge_of_merges_keeps_the_options_of_its_inner_shards(self): + same = {"websites": "none", "gitlab": "none"} + inner = merge.merge([write_shard(self.tmp, "s1", self.split[1], 1, 3), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example"))], + os.path.join(self.tmp, "inner"))["out"] + outer = merge.merge([inner, write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)], + os.path.join(self.tmp, "outer"), force=True)["out"] + self.assertEqual(prove._options_of(read(os.path.join(outer, "config.json"))), + [dict(same, gitlab="gitlab.example"), same, {}]) + def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): dirs = self.shards() os.makedirs(os.path.join(self.tmp, "taken", "x")) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267..ea68afc 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -6,6 +6,7 @@ import os import tempfile import unittest +from unittest import mock from nullius.standard import job from nullius.traces import trace @@ -139,6 +140,31 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) + def test_a_trace_that_fails_leaves_the_result_and_says_its_error(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + with mock.patch.object(job, "trace_from_run", side_effect=KeyError("the task's own words")): + path = self.write(AGENT, traj) + agent = os.path.dirname(path) + self.assertFalse(os.path.exists(path)) + with open(os.path.join(os.path.dirname(agent), "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["rewards"], {"reward": 1.0}) + with open(os.path.join(agent, "trace.error.txt")) as f: + self.assertEqual(f.read(), "KeyError\n") + self.assertTrue(os.path.exists(os.path.join(agent, "trajectory.json"))) + + def test_a_trace_error_that_cannot_be_written_leaves_the_result(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + write = job._write + + def refusing(path, text): + if os.path.basename(path).startswith("trace"): + raise PermissionError(path) + write(path, text) + + with mock.patch.object(job, "_write", refusing): + path = self.write(AGENT, traj) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(os.path.dirname(path)), "result.json"))) + class Backfill(unittest.TestCase): """A trial written before traces gets one from the files it kept, and nothing else changes.""" diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index d7c93da..594fd76 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -115,7 +115,7 @@ def tree(job_dir): mock.patch.object(install, "upstream", side_effect=install.InstallFailed("git fetch: exit 128")), \ self.assertRaises(run.Refused): preparing.start() - self.assertEqual(tree(preparing.job_dir), [".work", "progress.jsonl"]) + self.assertEqual(tree(preparing.job_dir), [".work", ".work/job.lck", "progress.jsonl"]) self.assertEqual(tree(unknown.job_dir), tree(preparing.job_dir)) self.assertIn("nope", progress.refusal(progress.read(unknown.job_dir))) @@ -347,6 +347,22 @@ def test_the_listed_units_run_under_the_machine_s_constraints(self): cfg = read(os.path.join(job, "config.json")) self.assertEqual((cfg["standard"]["units_file"]["units"], cfg["standard"]["constraints"], cfg["n_concurrent_trials"]), (["slow", "good"], {"concurrency": 1}, 1)) + def test_the_command_records_both_the_units_file_and_the_serving_pin(self): + path = self.file("good\n") + self.b["score"] = dict(self.b["score"], judge={"model": "m-judge", "says": "a judge"}) + # The fixture scores locally; no relay listener is needed. + for extra in ([], ["--from-run"]): + with self.subTest(extra=extra), mock.patch.object(run.relay.Relay, "start", lambda r: r), \ + mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9"), \ + mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"}): + code, out = self.cli("--units", path, "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", "--name", "pinned" + "".join(extra), *extra) + self.assertEqual(code, 0, out) + job = out.strip().splitlines()[-1] + standard = read(os.path.join(job, "config.json"))["standard"] + self.assertEqual(standard["units_file"], run.units_file(path)) + self.assertEqual(standard["served_by"], {"m-judge": ["prov-a"]}) + self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000"]) def test_a_list_the_run_cannot_take_is_refused_with_exit_2(self): for text, said in (("good\ngood\n", "more than once"), ("# none\n", "lists no unit"), ("good\nnope\n", "nope")): diff --git a/app/tests/test_standard_versions.py b/app/tests/test_standard_versions.py index 898a0dc..1c86087 100644 --- a/app/tests/test_standard_versions.py +++ b/app/tests/test_standard_versions.py @@ -217,7 +217,7 @@ def test_new_version_copies_the_default_and_refuses_a_duplicate(self): ''' TOY = dict(BENCH, upstream={"license": "MIT"}, default="1", controls={"nop": {"source": "nullius", "from": "nothing runs"}}, - versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 2}], + versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 3}], "driver": {"command": ["python3", "{here}/driver2.py"]}, "proven": {}}, "1": {"proven": {"all": {"units": {"units": 2, "passed": "2 ids", "on": "2026-10-12"}}}}}) diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 8be2862..364c22f 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -34,7 +34,7 @@ def test_every_standard_benchmark_draws_its_where_step_and_says_why_docker_is_no self.assertIn('title="Not on this machine: docker is not installed"', tile) self.assertEqual(html.count("Not on this machine"), 1) container = re.search(r'