From e4ad5f79e3c108415aaff58f537893d275c67824 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:49:18 +0000 Subject: [PATCH] test: cover multi-turn tool-call formatting in response_api message builders without API calls Co-Authored-By: Gilad Samuels --- tests/llm/test_response_api.py | 329 +++++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) diff --git a/tests/llm/test_response_api.py b/tests/llm/test_response_api.py index 6174f64de..03c315f01 100644 --- a/tests/llm/test_response_api.py +++ b/tests/llm/test_response_api.py @@ -16,6 +16,8 @@ OpenAIChatModelArgs, OpenAIResponseAPIMessageBuilder, OpenAIResponseModelArgs, + ToolCall, + ToolCalls, ) @@ -401,6 +403,333 @@ def test_openai_response_model_parse_and_cost(): assert global_tracker.stats["output_tokens"] == 40 +# --- Test multi-turn tool-call formatting (mocked, non-pricy) --- + + +def _make_openai_responses_computer_call_tool_calls(call_id="cu_call_1"): + computer_call = MagicMock() + computer_call.type = "computer_call" + computer_call.call_id = call_id + raw_response = create_mock_openai_responses_api_response() + raw_response.output.append(computer_call) + tool_call = ToolCall(name="click", arguments={"x": 1, "y": 2}, raw_call=computer_call) + return ToolCalls(tool_calls=[tool_call], raw_calls=raw_response) + + +def test_openai_response_model_multi_turn_tool_call_formatting(): + args = OpenAIResponseModelArgs(model_name="gpt-4.1") + first_response = create_mock_openai_responses_api_response( + outputs=[ + {"type": "reasoning", "summary": ["Need the weather tool."]}, + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + "call_id": "call_paris", + }, + ] + ) + second_response = create_mock_openai_responses_api_response( + outputs=[ + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Delhi"}', + "call_id": "call_delhi", + } + ] + ) + + with patch("agentlab.llm.response_api.OpenAI") as mock_openai_class: + mock_openai_class.return_value = MagicMock() + model = args.make_model() + builder = args.get_message_builder() + assert builder is OpenAIResponseAPIMessageBuilder + + messages = [builder.user().add_text("What is the weather in Paris?")] + with patch.object( + model.client.responses, "create", side_effect=[first_response, second_response] + ) as mock_create: + first_output = model(APIPayload(messages=messages, tools=responses_api_tools)) + assert len(first_output.tool_calls) == 1 + for tool_call in first_output.tool_calls: + tool_call.response_text("It's sunny! 25°C") + messages += [ + builder.add_responded_tool_calls(first_output.tool_calls), + builder.user().add_text("What is the weather in Delhi?"), + ] + second_output = model(APIPayload(messages=messages, tools=responses_api_tools)) + + assert mock_create.call_count == 2 + sent_input = mock_create.call_args_list[1].kwargs["input"] + assert sent_input == [ + { + "role": "user", + "content": [{"type": "input_text", "text": "What is the weather in Paris?"}], + }, + first_response.output[0], + first_response.output[1], + { + "type": "function_call_output", + "call_id": "call_paris", + "output": "It's sunny! 25°C", + }, + { + "role": "user", + "content": [{"type": "input_text", "text": "What is the weather in Delhi?"}], + }, + ] + assert second_output.action == "get_weather(location='Delhi')" + + +def test_openai_response_api_message_builder_computer_call_output(): + tool_calls = _make_openai_responses_computer_call_tool_calls(call_id="cu_call_1") + next(iter(tool_calls)).response_image("data:image/png;base64,SCREENSHOT") + + messages = OpenAIResponseAPIMessageBuilder.add_responded_tool_calls( + tool_calls + ).prepare_message() + + assert messages == [ + tool_calls.raw_calls.output[0], + { + "type": "computer_call_output", + "call_id": "cu_call_1", + "output": {"type": "input_image", "image_url": "data:image/png;base64,SCREENSHOT"}, + }, + ] + + +def test_openai_response_api_message_builder_rejects_image_in_function_call_response(): + raw_response = create_mock_openai_responses_api_response( + outputs=[ + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + "call_id": "call_paris", + } + ] + ) + tool_call = ToolCall( + name="get_weather", arguments={"location": "Paris"}, raw_call=raw_response.output[0] + ).response_image("data:image/png;base64,IMG") + tool_calls = ToolCalls(tool_calls=[tool_call], raw_calls=raw_response) + + msg = OpenAIResponseAPIMessageBuilder.add_responded_tool_calls(tool_calls) + with pytest.raises(AssertionError, match="Image output is not supported"): + msg.prepare_message() + + +def test_openai_response_api_message_builder_rejects_text_in_computer_call_response(): + tool_calls = _make_openai_responses_computer_call_tool_calls() + next(iter(tool_calls)).response_text("not a screenshot") + + msg = OpenAIResponseAPIMessageBuilder.add_responded_tool_calls(tool_calls) + with pytest.raises(AssertionError, match="Text output is not supported"): + msg.prepare_message() + + +def test_claude_model_multi_turn_tool_call_formatting(): + args = ClaudeResponseModelArgs(model_name="claude-3-haiku-20240307") + model = args.make_model() + builder = args.get_message_builder() + assert builder is AnthropicAPIMessageBuilder + + first_response = create_mock_anthropic_response( + text_content="Let me check.", + tool_use={"id": "toolu_paris", "name": "get_weather", "input": {"location": "Paris"}}, + ) + second_tool_use = MagicMock(spec=anthropic.types.ToolUseBlock) + second_tool_use.type = "tool_use" + second_tool_use.id = "toolu_rome" + second_tool_use.name = "get_weather" + second_tool_use.input = {"location": "Rome"} + first_response.content.append(second_tool_use) + second_response = create_mock_anthropic_response( + tool_use={"id": "toolu_delhi", "name": "get_weather", "input": {"location": "Delhi"}}, + ) + + messages = [builder.user().add_text("What is the weather in Paris and Rome?")] + with patch.object( + model.client.messages, "create", side_effect=[first_response, second_response] + ) as mock_create: + first_output = model(APIPayload(messages=messages, tools=anthropic_tools)) + assert len(first_output.tool_calls) == 2 + paris_call, rome_call = first_output.tool_calls + paris_call.response_text("Sunny, 25°C") + rome_call.response_text("Cloudy, 18°C") + messages += [ + builder.add_responded_tool_calls(first_output.tool_calls), + builder.user().add_text("What is the weather in Delhi?"), + ] + second_output = model(APIPayload(messages=messages, tools=anthropic_tools)) + + assert mock_create.call_count == 2 + sent_messages = mock_create.call_args_list[1].kwargs["messages"] + assert sent_messages == [ + { + "role": "user", + "content": [{"type": "text", "text": "What is the weather in Paris and Rome?"}], + }, + {"role": "assistant", "content": first_response.content}, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_paris", "content": "Sunny, 25°C"}, + {"type": "tool_result", "tool_use_id": "toolu_rome", "content": "Cloudy, 18°C"}, + ], + }, + { + "role": "user", + "content": [{"type": "text", "text": "What is the weather in Delhi?"}], + }, + ] + assert second_output.action == "get_weather(location='Delhi')" + + +def test_anthropic_api_message_builder_rejects_image_in_tool_response(): + raw_response = create_mock_anthropic_response( + tool_use={"id": "toolu_1", "name": "get_weather", "input": {"location": "Paris"}}, + ) + tool_call = ToolCall( + name="get_weather", arguments={"location": "Paris"}, raw_call=raw_response.content[0] + ).response_image("data:image/png;base64,IMG") + tool_calls = ToolCalls(tool_calls=[tool_call], raw_calls=raw_response) + + msg = AnthropicAPIMessageBuilder.add_responded_tool_calls(tool_calls) + with pytest.raises(AssertionError, match="Image output is not supported"): + msg.prepare_message() + + +def test_openai_chat_completion_model_multi_turn_tool_call_formatting(): + args = OpenAIChatModelArgs(model_name="gpt-4.1") + with patch("agentlab.llm.response_api.OpenAI") as mock_openai_class: + mock_openai_class.return_value = MagicMock() + model = args.make_model() + builder = args.get_message_builder() + assert builder is OpenAIChatCompletionAPIMessageBuilder + + first_response = create_mock_openai_chat_completion( + tool_calls=[ + { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location": "Paris"}'}, + }, + { + "id": "call_rome", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location": "Rome"}'}, + }, + ] + ) + second_response = create_mock_openai_chat_completion( + tool_calls=[ + { + "id": "call_delhi", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location": "Delhi"}'}, + } + ] + ) + + messages = [builder.user().add_text("What is the weather in Paris and Rome?")] + with patch.object( + model.client.chat.completions, "create", side_effect=[first_response, second_response] + ) as mock_create: + first_output = model(APIPayload(messages=messages, tools=chat_api_tools)) + assert len(first_output.tool_calls) == 2 + paris_call, rome_call = first_output.tool_calls + paris_call.response_text("Sunny, 25°C") + rome_call.response_text("Cloudy, 18°C") + messages += [ + builder.add_responded_tool_calls(first_output.tool_calls), + builder.user().add_text("What is the weather in Delhi?"), + ] + second_output = model(APIPayload(messages=messages, tools=chat_api_tools)) + + assert mock_create.call_count == 2 + sent_messages = mock_create.call_args_list[1].kwargs["messages"] + assert sent_messages == [ + { + "role": "user", + "content": [{"type": "text", "text": "What is the weather in Paris and Rome?"}], + }, + first_response.choices[0].message, + { + "name": "get_weather", + "role": "tool", + "tool_call_id": "call_paris", + "content": "Sunny, 25°C", + }, + { + "name": "get_weather", + "role": "tool", + "tool_call_id": "call_rome", + "content": "Cloudy, 18°C", + }, + { + "role": "user", + "content": [{"type": "text", "text": "What is the weather in Delhi?"}], + }, + ] + assert second_output.action == "get_weather(location='Delhi')" + + +def test_openai_chat_completion_api_message_builder_rejects_image_in_tool_response(): + raw_call = { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location": "Paris"}'}, + } + raw_response = create_mock_openai_chat_completion(tool_calls=[raw_call]) + tool_call = ToolCall( + name="get_weather", arguments={"location": "Paris"}, raw_call=raw_call + ).response_image("data:image/png;base64,IMG") + tool_calls = ToolCalls(tool_calls=[tool_call], raw_calls=raw_response) + + msg = OpenAIChatCompletionAPIMessageBuilder.add_responded_tool_calls(tool_calls) + with pytest.raises(AssertionError, match="Image output is not supported"): + msg.prepare_message() + + +TOOL_MESSAGE_BUILDERS = [ + OpenAIResponseAPIMessageBuilder, + AnthropicAPIMessageBuilder, + OpenAIChatCompletionAPIMessageBuilder, +] + + +@pytest.mark.parametrize("builder_cls", TOOL_MESSAGE_BUILDERS) +def test_add_responded_tool_calls_requires_all_responses(builder_cls): + answered = ToolCall(name="get_weather", arguments={"location": "Paris"}).response_text("Sunny") + unanswered = ToolCall(name="get_weather", arguments={"location": "Rome"}) + tool_calls = ToolCalls(tool_calls=[answered, unanswered]) + + with pytest.raises(AssertionError, match="All tool calls must have a response"): + builder_cls.add_responded_tool_calls(tool_calls) + + +@pytest.mark.parametrize("builder_cls", TOOL_MESSAGE_BUILDERS) +def test_add_responded_tool_calls_builds_tool_message(builder_cls): + tool_call = ToolCall(name="get_weather", arguments={"location": "Paris"}).response_text("Sunny") + tool_calls = ToolCalls(tool_calls=[tool_call]) + + msg = builder_cls.add_responded_tool_calls(tool_calls) + + assert isinstance(msg, builder_cls) + assert msg.role == "tool" + assert msg.responded_tool_calls is tool_calls + assert msg.content == [] + + +@pytest.mark.parametrize("builder_cls", TOOL_MESSAGE_BUILDERS) +def test_tool_message_without_responded_tool_calls_raises(builder_cls): + with pytest.raises(ValueError, match="No tool calls found"): + builder_cls("tool").prepare_message() + + # --- Test Response Models (Pricy - require API keys and actual calls) ---