diff --git a/CHANGELOG.md b/CHANGELOG.md index 9644db5..e6e6867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pinned b9066 builds predate MTP, upstream PR #22673). Example profile: `scripts/settings_local_llm_qwen38_27b.yaml` (unvalidated; numbers in it are labelled sourced / estimated). +- **fix(benchmarks): the in-loop runner now writes conversation logs** — with + `log_conversations: true` (the setting the CLI already honoured) `experiments/swebench_lite/ + run_in_loop.py` created no logger, so every benchmark trajectory was thrown away. It now writes + `/training/.conversation.jsonl` including the terminal `session_end` record + (exit reason and code), so benchmark runs build the trajectory corpus any later tuning needs. + - **feat(llm): Qwen3.5+ thinking control** — `qwen3.5`..`qwen3.9` models get `chat_template_kwargs` (`enable_thinking`, `reasoning_effort`) built from `reasoning_effort` (`none|low|medium|high`) or `thinking_budget` (mapped to diff --git a/experiments/swebench_lite/run_in_loop.py b/experiments/swebench_lite/run_in_loop.py index 2d99b47..70c39e9 100644 --- a/experiments/swebench_lite/run_in_loop.py +++ b/experiments/swebench_lite/run_in_loop.py @@ -63,7 +63,7 @@ async def _run_one_async( # Imports kept local so this module stays cheap to import from run.py from godspeed.agent.conversation import Conversation from godspeed.agent.loop import agent_loop - from godspeed.agent.result import AgentMetrics, ExitReason + from godspeed.agent.result import EXIT_REASON_TO_CODE, AgentMetrics, ExitCode, ExitReason from godspeed.audit.trail import AuditTrail from godspeed.cli import _ensure_ollama from godspeed.config import GodspeedSettings @@ -175,8 +175,19 @@ def evaluate(self, tool_call: Any) -> PermissionDecision: llm_client=llm_client, # type: ignore[arg-type] ) + # Benchmark runs generate exactly the trajectories a tuning corpus needs; honour the same + # log_conversations switch the CLI does (this runner used to drop them silently). + conversation_logger = None + if settings.log_conversations: + from godspeed.training.conversation_logger import ConversationLogger + + conversation_logger = ConversationLogger( + session_id=session_id, output_dir=settings.global_dir / "training" + ) + conversation = Conversation( system_prompt=system_prompt, + conversation_logger=conversation_logger, model=effective_model, max_tokens=settings.max_context_tokens, compaction_threshold=settings.compaction_threshold, @@ -234,6 +245,22 @@ def on_tool_call(name: str, _args: dict) -> None: timed_out = True final_text = f"(session exceeded wall-clock timeout of {timeout_s}s)" metrics.finalize(ExitReason.TIMEOUT) + except BaseException: + if conversation_logger is not None: + conversation_logger.close() + raise + + if conversation_logger is not None: + conversation_logger.log_session_end( + exit_reason=metrics.exit_reason.value, + exit_code=int(EXIT_REASON_TO_CODE.get(metrics.exit_reason, ExitCode.SUCCESS)), + iterations_used=metrics.iterations_used, + tool_call_count=metrics.tool_call_count, + tool_error_count=metrics.tool_error_count, + duration_seconds=round(metrics.duration_seconds, 3), + cost_usd=llm_client.total_cost_usd, + ) + conversation_logger.close() audit_trail.record( event_type="session_end", diff --git a/tests/test_run_in_loop_wiring.py b/tests/test_run_in_loop_wiring.py index b45354a..482125b 100644 --- a/tests/test_run_in_loop_wiring.py +++ b/tests/test_run_in_loop_wiring.py @@ -64,3 +64,55 @@ async def _fake_agent_loop(**_kwargs: Any) -> str: assert payload["final_text"] == "done" assert captured["reasoning_effort"] == "medium" assert captured["model"] == "openai/qwen3.8-27b" + + +def _run_with_settings( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, extra_yaml: str, stop_with: str = "done" +) -> Path: + """Run one in-loop session with a stub agent loop; return the global settings dir.""" + + async def _fake_agent_loop(**_kwargs: Any) -> str: + return stop_with + + monkeypatch.setattr("godspeed.agent.loop.agent_loop", _fake_agent_loop) + global_dir = tmp_path / "global" + global_dir.mkdir() + (global_dir / "settings.yaml").write_text( + f"global_dir: {global_dir.as_posix()}\n{extra_yaml}", encoding="utf-8" + ) + monkeypatch.setattr("godspeed.config.DEFAULT_GLOBAL_DIR", global_dir) + asyncio.run( + run_in_loop._run_one_async( + model="openai/qwen3.8-27b", + prompt="fix the bug", + project_dir=tmp_path, + instance_id="acme__widget-1", + split="dev", + timeout_s=30, + verify_workdir=tmp_path, + max_iterations=1, + ) + ) + return global_dir + + +def test_conversation_is_logged_when_enabled( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Benchmark runs produce the trajectories a tuning corpus needs; they must not be dropped.""" + import json + + global_dir = _run_with_settings(monkeypatch, tmp_path, "log_conversations: true\n") + files = list((global_dir / "training").glob("*.conversation.jsonl")) + assert len(files) == 1 + records = [json.loads(line) for line in files[0].read_text(encoding="utf-8").splitlines()] + assert records[0]["role"] == "system" + end = [r for r in records if r.get("role") == "session_end" or "exit_reason" in r] + assert end, records + assert end[-1]["exit_reason"] == "stopped" + assert end[-1]["exit_code"] == 0 + + +def test_no_conversation_log_when_disabled(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + global_dir = _run_with_settings(monkeypatch, tmp_path, "log_conversations: false\n") + assert not list((global_dir / "training").glob("*.conversation.jsonl"))