diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py index 7eb3b65..ec338cb 100644 --- a/src/microbots/auto_memory/__init__.py +++ b/src/microbots/auto_memory/__init__.py @@ -1 +1,13 @@ -# auto_memory — iterative agent loop with memory feedback \ No newline at end of file +"""Iterative agent loop with memory feedback (auto_memory package).""" + +from microbots.auto_memory.cli import run_from_yaml +from microbots.auto_memory.config import TaskConfig +from microbots.auto_memory.orchestrator import TrainingLoopOrchestrator +from microbots.auto_memory.runners.writing_bot_runner import WritingBotRunner + +__all__ = [ + "TrainingLoopOrchestrator", + "TaskConfig", + "WritingBotRunner", + "run_from_yaml", +] \ No newline at end of file diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py new file mode 100644 index 0000000..7f55124 --- /dev/null +++ b/src/microbots/auto_memory/cli.py @@ -0,0 +1,93 @@ +"""User-facing entry point that wires every auto_memory component.""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.callbacks import ShellCallbackRunner +from microbots.auto_memory.config import TaskConfig +from microbots.auto_memory.orchestrator import RunSummary, TrainingLoopOrchestrator +from microbots.auto_memory.runners.writing_bot_runner import WritingBotRunner +from microbots.auto_memory.workspace import WorkspaceManager + +logger = getLogger(__name__) + + +def run_from_yaml( + yaml_path: str | Path, + workdir: str | Path, + run_id: str | None = None, + *, + model: str, +) -> RunSummary: + """Load a task YAML, wire all components, and run the iteration loop. + + The run is materialised under ``/runs//`` with the + following on-disk layout:: + + /runs// + ├── memory/ + │ └── feedback.jsonl + ├── iterations/ + │ ├── iter_00/ + │ │ ├── candidate/ + │ │ └── logs/ + │ └── ... + └── run_meta.json + + Parameters + ---------- + yaml_path : str | Path + Path to the task configuration YAML file. + workdir : str | Path + Parent directory that holds the ``runs/`` tree. + run_id : str | None, optional + Identifier for this run. When ``None`` a UTC timestamp plus a short + random suffix of the form ``run-YYYYMMDD-HHMMSS-ffffff-`` is + generated to avoid collisions. + model : str + Model identifier forwarded to :class:`WritingBotRunner` (required, + keyword-only — e.g. ``"azure/gpt-4o"``). + + Returns + ------- + RunSummary + Summary of the completed run. + """ + config = TaskConfig.load_from_yaml(str(yaml_path)) + + if run_id is None: + run_id = _generate_run_id() + + run_dir = Path(workdir) / "runs" / run_id + logger.info("auto_memory: starting run %s at %s", run_id, run_dir) + + workspace = WorkspaceManager(run_dir=run_dir) + agent_runner = WritingBotRunner(model=model) + callback_runner = ShellCallbackRunner() + + orchestrator = TrainingLoopOrchestrator( + config=config, + agent_runner=agent_runner, + callback_runner=callback_runner, + workspace=workspace, + ) + + return orchestrator.run() + + +def _generate_run_id() -> str: + """Return a unique UTC-timestamp-based run identifier. + + Returns + ------- + str + Identifier of the form ``run-YYYYMMDD-HHMMSS-ffffff-`` using + the current UTC time plus an 8-character random suffix to guard + against collisions on coarse-resolution clocks or concurrent starts. + """ + timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S-%f") + return f"run-{timestamp}-{uuid.uuid4().hex[:8]}" diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py new file mode 100644 index 0000000..04d966d --- /dev/null +++ b/test/auto_memory/test_cli.py @@ -0,0 +1,178 @@ +"""End-to-end tests for the auto_memory CLI entry point.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory import run_from_yaml +from microbots.auto_memory.data_models import FinalStatus +from microbots.auto_memory.orchestrator import RunSummary +from microbots.MicroBot import BotRunResult + +_MODEL = "azure/gpt-4o" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_TASK_YAML = textwrap.dedent("""\ + task_definition: Write a hello message to /memories/hello.txt + prompt_template: "Goal: {{ task }}" + callbacks: + - name: always_ok + command: 'true' + max_iterations: 1 + timeout_min: 1 + per_iteration_timeout: 30 +""") + +_TASK_YAML_FAILING = textwrap.dedent("""\ + task_definition: Write a hello message to /memories/hello.txt + prompt_template: "Goal: {{ task }}" + callbacks: + - name: always_fail + command: 'false' + max_iterations: 2 + timeout_min: 1 + per_iteration_timeout: 30 +""") + + +def _write_yaml(tmp_path: Path, content: str = _TASK_YAML) -> Path: + p = tmp_path / "task.yml" + p.write_text(content) + return p + + +def _mock_writing_bot(status: bool = True, error: str | None = None): + """Patch WritingBot to return a controllable BotRunResult.""" + bot_instance = MagicMock() + bot_instance.run.return_value = BotRunResult( + status=status, + result="agent output" if status else None, + error=error, + ) + return patch( + "microbots.auto_memory.runners.writing_bot_runner.WritingBot", + return_value=bot_instance, + ), bot_instance + + +# --------------------------------------------------------------------------- +# End-to-end +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestRunFromYamlEndToEnd: + def test_returns_run_summary(self, tmp_path): + yaml_path = _write_yaml(tmp_path) + workdir = tmp_path / "workdir" + bot_patch, _ = _mock_writing_bot() + + with bot_patch, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ): + summary = run_from_yaml( + str(yaml_path), str(workdir), run_id="t1", model=_MODEL + ) + + assert isinstance(summary, RunSummary) + + def test_final_status_passed_when_callbacks_pass(self, tmp_path): + yaml_path = _write_yaml(tmp_path) + workdir = tmp_path / "workdir" + bot_patch, _ = _mock_writing_bot() + + with bot_patch, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ): + summary = run_from_yaml( + str(yaml_path), str(workdir), run_id="t2", model=_MODEL + ) + + assert summary.final_status == FinalStatus.PASSED + assert summary.iterations_run == 1 + assert summary.error_message is None + assert len(summary.iteration_records) == 1 + + def test_disk_layout_created(self, tmp_path): + yaml_path = _write_yaml(tmp_path) + workdir = tmp_path / "workdir" + bot_patch, _ = _mock_writing_bot() + + with bot_patch, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ): + run_from_yaml( + str(yaml_path), str(workdir), run_id="layout_run", model=_MODEL + ) + + run_dir = workdir / "runs" / "layout_run" + assert run_dir.is_dir() + assert (run_dir / "run_meta.json").is_file() + assert (run_dir / "memory").is_dir() + assert (run_dir / "memory" / "feedback.jsonl").is_file() + assert (run_dir / "iterations" / "iter_00").is_dir() + assert (run_dir / "iterations" / "iter_00" / "candidate").is_dir() + assert (run_dir / "iterations" / "iter_00" / "logs").is_dir() + + def test_writing_bot_receives_memory_dir(self, tmp_path): + yaml_path = _write_yaml(tmp_path) + workdir = tmp_path / "workdir" + bot_patch, bot_instance = _mock_writing_bot() + + with bot_patch as MockBot, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ) as MockMemoryTool: + run_from_yaml( + str(yaml_path), str(workdir), run_id="mem_run", model=_MODEL + ) + + expected_memory_dir = str(workdir / "runs" / "mem_run" / "memory") + _, kwargs = MockBot.call_args + assert kwargs["folder_to_mount"] == expected_memory_dir + MockMemoryTool.assert_called_once_with(memory_dir=expected_memory_dir) + + def test_auto_generated_run_id(self, tmp_path): + yaml_path = _write_yaml(tmp_path) + workdir = tmp_path / "workdir" + bot_patch, _ = _mock_writing_bot() + + with bot_patch, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ): + summary = run_from_yaml(str(yaml_path), str(workdir), model=_MODEL) + + assert summary.final_status == FinalStatus.PASSED + runs_dir = workdir / "runs" + children = [p for p in runs_dir.iterdir() if p.is_dir()] + assert len(children) == 1 + assert children[0].name.startswith("run-") + + def test_failing_callbacks_persist_feedback_and_reach_limit(self, tmp_path): + yaml_path = _write_yaml(tmp_path, _TASK_YAML_FAILING) + workdir = tmp_path / "workdir" + bot_patch, _ = _mock_writing_bot() + + with bot_patch, patch( + "microbots.auto_memory.runners.writing_bot_runner.MemoryTool" + ): + summary = run_from_yaml( + str(yaml_path), str(workdir), run_id="fail_run", model=_MODEL + ) + + assert summary.final_status == FinalStatus.LIMIT_REACHED + assert summary.iterations_run == 2 + + run_dir = workdir / "runs" / "fail_run" + feedback_file = run_dir / "memory" / "feedback.jsonl" + assert feedback_file.is_file() + lines = [ln for ln in feedback_file.read_text().splitlines() if ln.strip()] + assert len(lines) == 2 + assert (run_dir / "iterations" / "iter_01" / "candidate").is_dir()