Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions benchmarks/agentic/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
shortcut: the claude CLI is the harness (already installed, we run inside it). No SDK
dependency. The CLI's JSON output already carries cost/tokens/duration/permission_denials.
"""
import argparse, concurrent.futures, datetime, json, os, re, shutil, signal, statistics, subprocess, sys, tempfile, time, zlib
import argparse, concurrent.futures, datetime, json, os, re, shutil, signal, stat, statistics, subprocess, sys, tempfile, time, zlib
from collections import defaultdict
from pathlib import Path

Expand Down Expand Up @@ -283,6 +283,7 @@ def selftest():
failures += _selftest_session()
failures += _selftest_activation()
failures += _selftest_retries()
failures += _selftest_keep_cell()
failures += _selftest_isolated_scoring()
failures += _selftest_plugin_dir()
failures += _selftest_kill()
Expand Down Expand Up @@ -360,6 +361,17 @@ def flaky():
and len(calls) == len(RETRY_WAITS) + 1
return _report("retries a session error retries, then is reported", ok)

def _selftest_keep_cell():
"""A retry replaces the kept cell of the failed attempt, even with git's read-only objects in it."""
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as d:
dest, ws = Path(d) / "cell", Path(d) / "scratch" / "cell"
_write(dest / ".git" / "objects" / "ab" / "obj", "old attempt")
os.chmod(dest / ".git" / "objects" / "ab" / "obj", stat.S_IREAD)
_write(ws / "new.txt", "new attempt")
_keep_cell(ws, dest)
ok = (dest / "new.txt").is_file() and not (dest / "cell").exists() and not (dest / ".git").exists() and not ws.exists()
return _report("keep cell a retry replaces a kept cell holding read-only files", ok)

def _selftest_isolated_scoring():
"""Scoring one cell must not see another cell's code, and produced code that hangs or exits
must fail that cell without taking the run with it."""
Expand Down Expand Up @@ -672,6 +684,18 @@ def run_session(task_id, arm, model, workdir: Path):
shutil.move(str(log), str(workdir / "_claude.stream.jsonl"))
shutil.move(str(err), str(workdir / "_claude.stderr.txt"))

def _unlock(func, path, _error):
"""rmtree handler: git writes its objects read-only, and Windows will not delete a read-only file."""
os.chmod(path, stat.S_IWRITE)
func(path)

def _keep_cell(ws: Path, dest: Path):
"""Move a cell's workspace to dest, replacing a kept cell from an earlier attempt. A leftover that
cannot be removed raises: moving into it would nest the workspace and lose the paid session."""
if dest.exists():
shutil.rmtree(dest, **{"onexc" if sys.version_info >= (3, 12) else "onerror": _unlock})
shutil.move(str(ws), str(dest))

def _with_retries(run, wait=time.sleep):
"""Run a cell, retrying with backoff while its session fails: a rate limit is not the arm's fault."""
for pause in (*RETRY_WAITS, None):
Expand Down Expand Up @@ -844,8 +868,7 @@ def attempt():
try:
run_session(tid, arm, model, ws)
finally: # kept whatever happens: the session is paid for
shutil.rmtree(out_dir / name, ignore_errors=True) # a failed cell from an earlier attempt
shutil.move(str(ws), str(out_dir / name))
_keep_cell(ws, out_dir / name)
return score_isolated(tid, arm, model, out_dir / name)
return _with_retries(attempt)

Expand Down
Loading