Repository navigation
fix(trace): unnamed-agent ids, failed agent_end status, async model-call events #6086
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,11 @@ | |
| _turn_loop_guard = contextvars.ContextVar("praisonai_turn_loop_guard", default=None) | ||
|
|
||
|
|
||
| def _agent_end_status(db_status: str) -> str: | ||
| """``agent_end`` status for replay: completed, failed or cancelled.""" | ||
| return "failed" if db_status == "error" else db_status | ||
|
|
||
|
|
||
| def _enter_turn(agent, cancel): | ||
| """Mark a ``chat``/``achat`` turn in flight and bind its cancel token to this context.""" | ||
| with _turns_lock: | ||
|
|
@@ -3618,7 +3623,7 @@ def chat(self, prompt: str, temperature: Optional[float] = None, tools: Optional | |
| end_durable_run(durable_token) | ||
| durable_context.close() | ||
| try: | ||
| _trace_emitter.agent_end(self.name) | ||
| _trace_emitter.agent_end(self.name, {"status": _agent_end_status(db_status)}) | ||
| finally: | ||
| if getattr(self, "_active_turn_token", None) is _cancel: | ||
| self._active_turn_token = None | ||
|
|
@@ -4467,7 +4472,7 @@ async def achat(self, prompt: str, temperature: Optional[float] = None, tools: O | |
| await durable_context.aclose() | ||
| finally: | ||
| try: | ||
| _trace_emitter.agent_end(self.name) | ||
| _trace_emitter.agent_end(self.name, {"status": _agent_end_status(db_status)}) | ||
| finally: | ||
| if getattr(self, "_active_turn_token", None) is _cancel: | ||
| self._active_turn_token = None | ||
|
|
@@ -6775,6 +6780,57 @@ def _chat_completion_with_retry_core(self, messages, temperature=None, tools=Non | |
| raise RuntimeError("Retry loop completed without returning or raising an exception") | ||
|
|
||
| async def _achat_completion_with_retry(self, messages, temperature=None, tools=None, stream=None, reasoning_steps=False, task_name=None, task_description=None, task_id=None, response_format=None, stream_callback=None, emit_events=True): | ||
| """Async model call with retry, recorded as ``llm_request``/``llm_response`` like the sync path. | ||
|
|
||
| ``Agent.astart``/``achat`` recorded no model calls before this, so a replay | ||
| of an async run showed the agent but never what it asked the model. | ||
| """ | ||
| from ..trace.context_events import get_context_emitter | ||
|
|
||
| _trace_emitter = get_context_emitter() | ||
| call = self._achat_completion_with_retry_dispatch( | ||
| messages, temperature, tools, stream, reasoning_steps, task_name, | ||
| task_description, task_id, response_format, | ||
| stream_callback=stream_callback, emit_events=emit_events, | ||
| ) | ||
| if not _trace_emitter._enabled: # zero overhead when tracing is off | ||
| return await call | ||
| _trace_emitter.llm_request( | ||
| self.name, | ||
| messages_count=len(messages), | ||
| tokens_used=0, | ||
| model=self.llm if isinstance(self.llm, str) else None, | ||
| messages=messages, | ||
| ) | ||
| started = time.time() | ||
| try: | ||
| response = await call | ||
| except Exception as e: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Close the model-call trace on task cancellation. If the task is cancelled while awaiting
|
||
| _trace_emitter.llm_response( | ||
| self.name, | ||
| duration_ms=(time.time() - started) * 1000, | ||
| finish_reason="error", | ||
| response_content=str(e), | ||
| ) | ||
| raise | ||
| prompt_tokens = completion_tokens = 0 | ||
| cost_usd = 0.0 | ||
| usage = getattr(response, "usage", None) | ||
| if usage: | ||
| prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0 | ||
| completion_tokens = getattr(usage, "completion_tokens", 0) or 0 | ||
| cost_usd = self._calculate_llm_cost(prompt_tokens, completion_tokens, response=response) | ||
|
Comment on lines
+6818
to
+6822
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For async runs with tools, |
||
| _trace_emitter.llm_response( | ||
| self.name, | ||
| duration_ms=(time.time() - started) * 1000, | ||
| response_content=self._extract_llm_response_content(response), | ||
| prompt_tokens=prompt_tokens, | ||
| completion_tokens=completion_tokens, | ||
| cost_usd=cost_usd, | ||
| ) | ||
| return response | ||
|
|
||
| async def _achat_completion_with_retry_dispatch(self, messages, temperature=None, tools=None, stream=None, reasoning_steps=False, task_name=None, task_description=None, task_id=None, response_format=None, stream_callback=None, emit_events=True): | ||
| """ | ||
| Async wrapper for _execute_unified_achat_completion that adds jittered exponential backoff retry logic. | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| """Unnamed agents in traces and graphs, agent_end status, and async model-call events. | ||
|
|
||
| Deterministic: the model call is replaced, so nothing reaches a provider. | ||
| """ | ||
|
|
||
| import asyncio | ||
| from types import SimpleNamespace | ||
| from unittest.mock import patch | ||
|
|
||
| import pytest | ||
|
|
||
| from praisonaiagents.trace.context_events import ( | ||
| ContextListSink, | ||
| ContextTraceEmitter, | ||
| reset_context_emitter, | ||
| set_context_emitter, | ||
| ) | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def sink(monkeypatch): | ||
| monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-used") | ||
| s = ContextListSink() | ||
| token = set_context_emitter(ContextTraceEmitter(sink=s, session_id="t", enabled=True)) | ||
| yield s | ||
| reset_context_emitter(token) | ||
|
|
||
|
|
||
| def _of(sink, kind): | ||
| return [e for e in sink.get_events() if e.event_type.value == kind] | ||
|
|
||
|
|
||
| def _fake_completion(text="Solar sails ride sunlight."): | ||
| message = SimpleNamespace(content=text, tool_calls=None, role="assistant", reasoning_content=None) | ||
| return SimpleNamespace( | ||
| choices=[SimpleNamespace(message=message, finish_reason="stop", index=0)], | ||
| usage=SimpleNamespace(prompt_tokens=11, completion_tokens=7, total_tokens=18), | ||
| model="gpt-4o-mini", | ||
| ) | ||
|
|
||
|
|
||
| def search(query: str) -> str: | ||
| """Search the web.""" | ||
| return "result" | ||
|
|
||
|
|
||
| # --- to_graph and ids --------------------------------------------------------- | ||
|
|
||
| def test_to_graph_accepts_an_unnamed_agent(monkeypatch): | ||
| monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-used") | ||
| from praisonaiagents import Agent | ||
| from praisonaiagents.workflows.diagram import to_graph | ||
|
|
||
| graph = to_graph(Agent(instructions="Answer briefly.", tools=[search])) | ||
| ids = [n["id"] for n in graph["nodes"]] | ||
| assert ids[:3] == ["start:begin", "agent:Agent", "tool:Agent/search"] or set(ids) >= { | ||
| "start:begin", "agent:Agent", "tool:Agent/search", "end:done"} | ||
| assert next(n for n in graph["nodes"] if n["id"] == "agent:Agent")["label"] == "Agent" | ||
|
|
||
|
|
||
| def test_named_agent_graph_is_unchanged(monkeypatch): | ||
| monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-used") | ||
| from praisonaiagents import Agent | ||
| from praisonaiagents.workflows.diagram import to_graph | ||
|
|
||
| ids = [n["id"] for n in to_graph(Agent(name="Writer", instructions="x", tools=[search]))["nodes"]] | ||
| assert "agent:Writer" in ids and "tool:Writer/search" in ids | ||
|
|
||
|
|
||
| def test_emitter_ids_for_an_unnamed_agent(): | ||
| s = ContextListSink() | ||
| em = ContextTraceEmitter(sink=s, session_id="t", enabled=True) | ||
| em.agent_start(None) | ||
| em.tool_call_start(None, "search", {"query": "x"}) | ||
| em.tool_call_end(None, "search", "ok", 1.0) | ||
| em.agent_end(None) | ||
| assert [e.node_id for e in s.get_events()] == [ | ||
| "agent:Agent", "tool:Agent/search", "tool:Agent/search", "agent:Agent"] | ||
|
|
||
|
|
||
| def test_unnamed_agent_run_matches_its_graph(sink): | ||
| from praisonaiagents import Agent | ||
| from praisonaiagents.workflows.diagram import to_graph | ||
|
|
||
| agent = Agent(instructions="Answer briefly.") | ||
| with patch.object(agent, "_chat_completion", return_value=_fake_completion()): | ||
| agent.chat("Why do solar sails work?") | ||
| graph_ids = {n["id"] for n in to_graph(agent)["nodes"]} | ||
| for kind in ("agent_start", "agent_end"): | ||
| (event,) = _of(sink, kind) | ||
| assert event.agent_name is None # the name is not invented on the event | ||
| assert event.node_id == "agent:Agent" and event.node_id in graph_ids | ||
|
|
||
|
|
||
| # --- agent_end status --------------------------------------------------------- | ||
|
|
||
| def test_agent_end_records_completed(sink): | ||
| from praisonaiagents import Agent | ||
|
|
||
| agent = Agent(name="Writer", instructions="x") | ||
| with patch.object(agent, "_chat_completion", return_value=_fake_completion()): | ||
| assert agent.chat("hi") | ||
| assert _of(sink, "agent_end")[0].data == {"status": "completed"} | ||
|
|
||
|
|
||
| def test_agent_end_records_failed(sink): | ||
| from praisonaiagents import Agent | ||
|
|
||
| agent = Agent(name="Writer", instructions="x") | ||
| with patch.object(agent, "_chat_completion", side_effect=RuntimeError("provider down")): | ||
| try: | ||
| result = agent.chat("hi") | ||
| except Exception: | ||
| result = None | ||
| assert result is None | ||
| assert _of(sink, "agent_end")[0].data == {"status": "failed"} | ||
|
|
||
|
|
||
| # --- async model calls -------------------------------------------------------- | ||
|
|
||
| def test_astart_records_the_model_call(sink): | ||
| from praisonaiagents import Agent | ||
|
|
||
| agent = Agent(name="Writer", instructions="x", llm="gpt-4o-mini") | ||
|
|
||
| async def fake_dispatch(*args, **kwargs): | ||
| return _fake_completion() | ||
|
|
||
| with patch.object(agent, "_achat_completion_with_retry_dispatch", side_effect=fake_dispatch): | ||
| answer = asyncio.run(agent.astart("Why do solar sails work?")) | ||
| assert "sunlight" in str(answer) | ||
| (req,) = _of(sink, "llm_request") | ||
| (resp,) = _of(sink, "llm_response") | ||
| assert req.node_id == resp.node_id == "agent:Writer" | ||
| assert resp.prompt_tokens == 11 and resp.completion_tokens == 7 and resp.cost_usd > 0 | ||
| assert "sunlight" in (resp.data.get("response_content") or "") | ||
| assert _of(sink, "agent_end")[0].data == {"status": "completed"} | ||
|
|
||
|
|
||
| def test_async_model_error_is_recorded(sink): | ||
| from praisonaiagents import Agent | ||
|
|
||
| agent = Agent(name="Writer", instructions="x", llm="gpt-4o-mini") | ||
|
|
||
| async def boom(*args, **kwargs): | ||
| raise RuntimeError("rate limited") | ||
|
|
||
| with patch.object(agent, "_achat_completion_with_retry_dispatch", side_effect=boom): | ||
| try: | ||
| asyncio.run(agent.astart("hi")) | ||
| except Exception: | ||
| pass | ||
| responses = _of(sink, "llm_response") | ||
| assert responses and responses[0].data.get("finish_reason") == "error" | ||
| assert "rate limited" in (responses[0].data.get("response_content") or "") | ||
|
|
||
|
|
||
| def test_async_path_records_nothing_when_tracing_is_off(monkeypatch): | ||
| monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-used") | ||
| from praisonaiagents import Agent | ||
|
|
||
| s = ContextListSink() | ||
| token = set_context_emitter(ContextTraceEmitter(sink=s, session_id="t", enabled=False)) | ||
| try: | ||
| agent = Agent(name="Writer", instructions="x", llm="gpt-4o-mini") | ||
|
|
||
| async def fake_dispatch(*args, **kwargs): | ||
| return _fake_completion() | ||
|
|
||
| with patch.object(agent, "_achat_completion_with_retry_dispatch", side_effect=fake_dispatch) as d: | ||
| asyncio.run(agent.astart("hi")) | ||
| assert d.called and s.get_events() == [] | ||
| finally: | ||
| reset_context_emitter(token) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Trace the request after
BEFORE_LLMredaction.When a
BEFORE_LLMhook redacts PII, this call gives the trace sink the originalmessagesbefore_execute_unified_achat_completionapplies the hook.ContextTraceEmitterdoes not redactllm_requestmessages itself. A sink that writes on receipt can therefore retain text that the provider never receives. Emit the request from the post-hook dispatch path, using the messages actually sent to the provider. (raw.githubusercontent.com)