diff --git a/include/ryzenai/tool_calls.h b/include/ryzenai/tool_calls.h index 837c778..e053c17 100644 --- a/include/ryzenai/tool_calls.h +++ b/include/ryzenai/tool_calls.h @@ -22,5 +22,10 @@ std::pair, std::string> extractToolCalls(const std::string // Format tool calls in OpenAI API format json formatToolCallsForOpenAI(const std::vector& tool_calls); +// Build chat.completion.chunk JSON payloads (without the "data: " prefix) for +// streaming tool_calls deltas. Uses the same OpenAI shape as formatToolCallsForOpenAI. +std::vector buildToolCallStreamChunks(const json& openai_tool_calls, + const std::string& model_id); + } // namespace ryzenai diff --git a/src/server.cpp b/src/server.cpp index 3f2c36e..c3c2cef 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -12,6 +12,33 @@ namespace ryzenai { namespace { +std::string escapeJsonString(const std::string& s) { + static constexpr char hex[] = "0123456789abcdef"; + std::string escaped; + escaped.reserve(s.size() + 8); + for (unsigned char c : s) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\b': escaped += "\\b"; break; + case '\f': escaped += "\\f"; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: + if (c < 0x20) { + escaped += "\\u00"; + escaped += hex[(c >> 4) & 0x0f]; + escaped += hex[c & 0x0f]; + } else { + escaped.push_back(static_cast(c)); + } + break; + } + } + return escaped; +} + // Minimal base64 decoder for image data URLs. std::string base64_decode(const std::string& input) { static const std::string chars = @@ -316,7 +343,7 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp ); std::string prompt = comp_req.prompt; - std::string model_id = model_id_; + std::string model_id = escapeJsonString(model_id_); // Count prompt tokens before streaming int prompt_tokens = inference_engine_->countTokens(prompt); @@ -348,30 +375,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -452,30 +457,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // After generation completes, do a final flush to catch any remaining buffered content auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -675,6 +658,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: ); std::string model_id = model_id_; + std::string escaped_model_id = escapeJsonString(model_id); bool has_tools = !chat_req.tools.empty(); // Count prompt tokens before streaming @@ -682,7 +666,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: res.set_chunked_content_provider( "text/event-stream", - [this, prompt, params, model_id, has_tools, prompt_tokens](size_t offset, httplib::DataSink& sink) { + [this, prompt, params, model_id, escaped_model_id, has_tools, prompt_tokens](size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; // Only run once try { @@ -700,7 +684,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Generate and send tokens in real-time inference_engine_->streamComplete(prompt, params, - [&sink, model_id, &token_count, &full_response, &reasoning_parser, &first_token_received, &first_token_time](const std::string& token, bool is_final) -> bool { + [&sink, escaped_model_id, &token_count, &full_response, &reasoning_parser, &first_token_received, &first_token_time](const std::string& token, bool is_final) -> bool { // Track time to first token if (!first_token_received && !token.empty()) { first_token_time = std::chrono::high_resolution_clock::now(); @@ -713,30 +697,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -745,7 +707,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string reasoning_chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"" + escaped_reasoning + "\"},\"finish_reason\":null}]}"; @@ -762,7 +724,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"" + escaped_content + "\"},\"finish_reason\":" + finish_reason + "}]}"; @@ -782,7 +744,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string reasoning_chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"" + escaped_reasoning + "\"},\"finish_reason\":null}]}"; @@ -798,7 +760,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"" + escaped_content + "\"},\"finish_reason\":\"stop\"}]}"; @@ -818,30 +780,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // This handles the case where the last few tokens didn't trigger processing due to buffer size auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings (reused from above) auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -850,7 +790,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string reasoning_chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"" + escaped_reasoning + "\"},\"finish_reason\":null}]}"; @@ -864,7 +804,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string chunk_json = "{\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"" + escaped_content + "\"},\"finish_reason\":\"stop\"}]}"; @@ -874,34 +814,15 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Extract and send tool calls if tools were provided if (has_tools) { - auto [extracted_tool_calls, cleaned_text] = extractToolCalls(full_response); + auto extraction = extractToolCalls(full_response); + const auto& extracted_tool_calls = extraction.first; if (!extracted_tool_calls.empty()) { std::cout << "[Server] Extracted " << extracted_tool_calls.size() << " tool call(s) from stream" << std::endl; - - // Send tool calls as delta chunks - for (const auto& tool_call : extracted_tool_calls) { - // Escape arguments for JSON - std::string tool_call_args = tool_call.arguments.dump(); - std::string escaped_args = tool_call_args; - size_t pos = 0; - while ((pos = escaped_args.find('\\', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_args.find('"', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\""); - pos += 2; - } - - std::string tool_call_chunk = - "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + - "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + - "\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"-\",\"type\":\"function\",\"function\":{\"name\":\"" + tool_call.name + - "\",\"arguments\":\"" + escaped_args + "\"}}]},\"finish_reason\":null}]}\n\n"; - - sink.write(tool_call_chunk.c_str(), tool_call_chunk.size()); + + json openai_tool_calls = formatToolCallsForOpenAI(extracted_tool_calls); + for (const std::string& chunk_json : buildToolCallStreamChunks(openai_tool_calls, model_id)) { + std::string chunk_str = "data: " + chunk_json + "\n\n"; + sink.write(chunk_str.c_str(), chunk_str.size()); } } } @@ -918,7 +839,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string usage_chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":null}]," + "\"usage\":{" + "\"prompt_tokens\":" + std::to_string(prompt_tokens) + "," + @@ -1065,22 +986,11 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, chat_req.top_k, chat_req.repeat_penalty, chat_req.stop); auto escapeJson = [](const std::string& str) -> std::string { - std::string e; - e.reserve(str.size() + 8); - for (char c : str) { - switch (c) { - case '\\': e += "\\\\"; break; - case '"': e += "\\\""; break; - case '\n': e += "\\n"; break; - case '\r': e += "\\r"; break; - case '\t': e += "\\t"; break; - default: e += c; break; - } - } - return e; + return escapeJsonString(str); }; const std::string model_id = model_id_; + const std::string escaped_model_id = escapeJsonString(model_id); if (chat_req.stream) { res.set_header("Content-Type", "text/event-stream"); @@ -1092,16 +1002,16 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, res.set_chunked_content_provider( "text/event-stream", - [this, prompt, images, params, model_id, prompt_tokens, escapeJson](size_t offset, httplib::DataSink& sink) { + [this, prompt, images, params, escaped_model_id, prompt_tokens, escapeJson](size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; try { int token_count = 0; inference_engine_->streamCompleteMultimodal(prompt, images, params, - [&sink, model_id, &token_count, &escapeJson](const std::string& token, bool /*is_final*/) -> bool { + [&sink, escaped_model_id, &token_count, &escapeJson](const std::string& token, bool /*is_final*/) -> bool { std::string chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"" + escapeJson(token) + "\"},\"finish_reason\":null}]}\n\n"; token_count++; @@ -1111,7 +1021,7 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, std::string final_chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + "\",\"object\":\"chat.completion.chunk\",\"created\":" + std::to_string(std::time(nullptr)) + - ",\"model\":\"" + model_id + + ",\"model\":\"" + escaped_model_id + "\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]," + "\"usage\":{\"prompt_tokens\":" + std::to_string(prompt_tokens) + ",\"completion_tokens\":" + std::to_string(token_count) + @@ -1225,7 +1135,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon top_k, repeat_penalty, {} ); - std::string model_name = model; + std::string model_name = escapeJsonString(model); res.set_chunked_content_provider( "text/event-stream", @@ -1253,28 +1163,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon // Generate and send tokens in real-time inference_engine_->streamComplete(prompt, params, [&sink, &full_response](const std::string& token, bool is_final) -> bool { - // Escape special characters for JSON - std::string escaped_token = token; - size_t pos = 0; - while ((pos = escaped_token.find('\\', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('"', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\n', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\r', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_token = escapeJsonString(token); // Accumulate unescaped token for final response full_response += token; @@ -1295,28 +1184,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon std::cout << "[Server] Token generation completed, sending final events" << std::endl; - // Escape full_response for JSON - std::string escaped_full_response = full_response; - size_t pos = 0; - while ((pos = escaped_full_response.find('\\', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('"', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\n', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\r', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_full_response = escapeJsonString(full_response); // Send response.completed event std::string completed_time = std::to_string(std::time(nullptr)); @@ -1347,7 +1215,8 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon } catch (const std::exception& e) { std::cerr << "[Server] Error in streaming responses: " << e.what() << std::endl; - std::string error_msg = "data: {\"error\":\"" + std::string(e.what()) + "\"}\n\n"; + json error_chunk = createErrorResponse(e.what(), "inference_error"); + std::string error_msg = "data: " + error_chunk.dump() + "\n\n"; sink.write(error_msg.c_str(), error_msg.size()); return false; } diff --git a/src/tool_calls.cpp b/src/tool_calls.cpp index 83e769e..215208d 100644 --- a/src/tool_calls.cpp +++ b/src/tool_calls.cpp @@ -1,153 +1,417 @@ #include "ryzenai/tool_calls.h" #include #include +#include +#include namespace ryzenai { +namespace { + +std::string trimWhitespace(const std::string& input) { + size_t start = input.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) { + return ""; + } + size_t end = input.find_last_not_of(" \t\n\r"); + return input.substr(start, end - start + 1); +} + +bool isValidJsonEscapeSequence(const std::string& input, size_t backslash_index) { + if (backslash_index + 1 >= input.size()) { + return false; + } + const char next = input[backslash_index + 1]; + switch (next) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + return true; + case 'u': + // \u must be followed by exactly 4 hex digits; bare \users is not valid JSON. + if (backslash_index + 5 >= input.size()) { + return false; + } + for (size_t j = backslash_index + 2; j < backslash_index + 6; ++j) { + if (!std::isxdigit(static_cast(input[j]))) { + return false; + } + } + return true; + default: + return false; + } +} + +// LLMs often emit literal newlines or Windows paths with single backslashes inside +// JSON string values. Repair those before json::parse. +std::string sanitizeJsonStringLiterals(const std::string& input) { + std::string result; + result.reserve(input.size() + 32); + + bool in_string = false; + bool escape_next = false; + + for (size_t i = 0; i < input.size(); ++i) { + char c = input[i]; + + if (escape_next) { + result += c; + escape_next = false; + continue; + } + + if (!in_string) { + if (c == '"') { + in_string = true; + } + result += c; + continue; + } + + // Inside a JSON string value + if (c == '\\') { + if (isValidJsonEscapeSequence(input, i)) { + result += c; + escape_next = true; + } else { + result += "\\\\"; + } + } else if (c == '"') { + in_string = false; + result += c; + } else if (c == '\n') { + result += "\\n"; + } else if (c == '\r') { + result += "\\r"; + } else if (c == '\t') { + result += "\\t"; + } else { + result += c; + } + } + + return result; +} + +std::string balanceJsonBrackets(const std::string& input) { + int brace_depth = 0; + int bracket_depth = 0; + bool in_string = false; + bool escape_next = false; + + for (char c : input) { + if (escape_next) { + escape_next = false; + continue; + } + + if (in_string) { + if (c == '\\') { + escape_next = true; + } else if (c == '"') { + in_string = false; + } + continue; + } + + if (c == '"') { + in_string = true; + } else if (c == '{') { + ++brace_depth; + } else if (c == '}') { + --brace_depth; + } else if (c == '[') { + ++bracket_depth; + } else if (c == ']') { + --bracket_depth; + } + } + + std::string result = input; + while (bracket_depth > 0) { + result += ']'; + --bracket_depth; + } + while (brace_depth > 0) { + result += '}'; + --brace_depth; + } + return result; +} + +json tryParseJson(const std::string& text) { + try { + return json::parse(text); + } catch (const json::exception&) { + return json(); + } +} + +json parseToolCallJsonObject(const std::string& raw) { + const std::string trimmed = trimWhitespace(raw); + if (trimmed.empty()) { + return json(); + } + + json result = tryParseJson(trimmed); + if (!result.is_null()) { + return result; + } + + const std::string sanitized = sanitizeJsonStringLiterals(trimmed); + result = tryParseJson(sanitized); + if (!result.is_null()) { + std::cout << "[ToolCalls DEBUG] Parsed after sanitizing JSON string literals" << std::endl; + return result; + } + + const std::string balanced = balanceJsonBrackets(sanitized); + result = tryParseJson(balanced); + if (!result.is_null()) { + std::cout << "[ToolCalls DEBUG] Parsed after balancing JSON brackets" << std::endl; + return result; + } + + return json(); +} + +json normalizeToolCallArguments(const json& arguments) { + if (arguments.is_string()) { + const std::string raw = arguments.get(); + json parsed = parseToolCallJsonObject(raw); + if (!parsed.is_null()) { + return parsed; + } + try { + return json::parse(raw); + } catch (const json::exception&) { + return json{{"raw", raw}}; + } + } + return arguments; +} + +std::string toolCallArgumentsString(const json& arguments) { + if (arguments.is_string()) { + return arguments.get(); + } + if (arguments.is_object() || arguments.is_array()) { + return arguments.dump(); + } + return arguments.dump(); +} + +std::optional toolCallFromJsonObject(const json& tool_call_obj) { + if (!tool_call_obj.is_object()) { + return std::nullopt; + } + + ToolCall tool_call; + const json* fields = &tool_call_obj; + + if (tool_call_obj.contains("function") && tool_call_obj["function"].is_object()) { + fields = &tool_call_obj["function"]; + } + + if (fields->contains("name") && (*fields)["name"].is_string()) { + tool_call.name = (*fields)["name"]; + } else { + std::cerr << "[WARNING] Tool call missing 'name' field, skipping" << std::endl; + return std::nullopt; + } + + if (fields->contains("arguments")) { + tool_call.arguments = normalizeToolCallArguments((*fields)["arguments"]); + } else if (fields->contains("parameters")) { + tool_call.arguments = normalizeToolCallArguments((*fields)["parameters"]); + } else { + std::cerr << "[WARNING] Tool call missing 'arguments' or 'parameters' field, skipping" << std::endl; + return std::nullopt; + } + + return tool_call; +} + +std::optional parseToolCallPayload(const std::string& tool_call_json) { + json tool_call_obj = parseToolCallJsonObject(tool_call_json); + if (tool_call_obj.is_null()) { + return std::nullopt; + } + return toolCallFromJsonObject(tool_call_obj); +} + +void eraseMatch(std::string& cleaned_text, size_t match_pos, size_t match_len) { + if (match_pos < cleaned_text.size()) { + cleaned_text.erase(match_pos, std::min(match_len, cleaned_text.size() - match_pos)); + } +} + +} // namespace + std::pair, std::string> extractToolCalls(const std::string& text) { std::vector tool_calls; std::string cleaned_text = text; - + std::cout << "[ToolCalls DEBUG] Extracting tool calls from text (" << text.length() << " chars)" << std::endl; std::cout << "[ToolCalls DEBUG] Text: " << text.substr(0, std::min(size_t(300), text.length())) << std::endl; - - // Pattern for Qwen-style tool calls: ... - // Use [\s\S]*? to match across newlines (. doesn't match newlines in C++ regex by default) - std::regex tool_call_pattern(R"(([\s\S]*?))", std::regex::icase | std::regex::ECMAScript); - + + // Qwen-style tool calls: ... + const std::regex closed_tool_call_pattern( + R"(([\s\S]*?))", + std::regex::icase | std::regex::ECMAScript); + std::smatch match; std::string search_text = text; size_t offset = 0; - - while (std::regex_search(search_text, match, tool_call_pattern)) { - std::string tool_call_json = match[1].str(); - + size_t removed = 0; + + while (std::regex_search(search_text, match, closed_tool_call_pattern)) { + const std::string tool_call_json = match[1].str(); + std::cout << "[ToolCalls DEBUG] Found Qwen-style match, JSON: " << tool_call_json << std::endl; - - try { - // Parse the tool call JSON - json tool_call_obj = json::parse(tool_call_json); - - ToolCall tool_call; - - // Extract name - if (tool_call_obj.contains("name") && tool_call_obj["name"].is_string()) { - tool_call.name = tool_call_obj["name"]; - } else { - std::cerr << "[WARNING] Tool call missing 'name' field, skipping" << std::endl; - search_text = match.suffix(); - offset += match.position() + match.length(); - continue; - } - - // Extract arguments (can be "arguments" or "parameters") - if (tool_call_obj.contains("arguments")) { - tool_call.arguments = tool_call_obj["arguments"]; - } else if (tool_call_obj.contains("parameters")) { - tool_call.arguments = tool_call_obj["parameters"]; - } else { - std::cerr << "[WARNING] Tool call missing 'arguments' or 'parameters' field, skipping" << std::endl; - search_text = match.suffix(); - offset += match.position() + match.length(); - continue; - } - - tool_calls.push_back(tool_call); - - // Remove the tool call from the cleaned text - size_t match_pos = offset + match.position(); - size_t match_len = match.length(); - cleaned_text.erase(match_pos, match_len); - - } catch (const json::exception& e) { - std::cerr << "[WARNING] Failed to parse tool call JSON: " << e.what() << std::endl; + + if (auto tool_call = parseToolCallPayload(tool_call_json)) { + tool_calls.push_back(*tool_call); + eraseMatch(cleaned_text, offset + match.position() - removed, match.length()); + removed += match.length(); + } else { + std::cerr << "[WARNING] Failed to parse tool call JSON after repair attempts" << std::endl; } - + search_text = match.suffix(); offset += match.position() + match.length(); } - - // Also check for [TOOL_CALLS] [...] format (Mistral-style) - std::regex mistral_pattern(R"(\[TOOL_CALLS\]\s*\[([\s\S]*?)\])", std::regex::icase | std::regex::ECMAScript); + + // Handle truncated generations that open but never emit . + const std::regex open_tool_call_pattern( + R"(([\s\S]+)$)", + std::regex::icase | std::regex::ECMAScript); + search_text = cleaned_text; offset = 0; - + removed = 0; + while (std::regex_search(search_text, match, open_tool_call_pattern)) { + const std::string tool_call_json = match[1].str(); + + std::cout << "[ToolCalls DEBUG] Found unclosed Qwen-style match, JSON: " << tool_call_json << std::endl; + + if (auto tool_call = parseToolCallPayload(tool_call_json)) { + tool_calls.push_back(*tool_call); + eraseMatch(cleaned_text, offset + match.position() - removed, match.length()); + removed += match.length(); + } else { + std::cerr << "[WARNING] Failed to parse unclosed tool call JSON after repair attempts" << std::endl; + } + + search_text = match.suffix(); + offset += match.position() + match.length(); + } + + // Mistral-style: [TOOL_CALLS] [...] + const std::regex mistral_pattern( + R"(\[TOOL_CALLS\]\s*\[([\s\S]*?)\])", + std::regex::icase | std::regex::ECMAScript); + + search_text = cleaned_text; + offset = 0; + removed = 0; + while (std::regex_search(search_text, match, mistral_pattern)) { - std::string tool_calls_array_json = "[" + match[1].str() + "]"; - - try { - json tool_calls_array = json::parse(tool_calls_array_json); - - if (tool_calls_array.is_array()) { - for (const auto& tool_call_obj : tool_calls_array) { - ToolCall tool_call; - - if (tool_call_obj.contains("name") && tool_call_obj["name"].is_string()) { - tool_call.name = tool_call_obj["name"]; - } else { - continue; - } - - if (tool_call_obj.contains("arguments")) { - tool_call.arguments = tool_call_obj["arguments"]; - } else if (tool_call_obj.contains("parameters")) { - tool_call.arguments = tool_call_obj["parameters"]; - } else { - continue; - } - - tool_calls.push_back(tool_call); + const std::string tool_calls_array_json = "[" + match[1].str() + "]"; + + json tool_calls_array = parseToolCallJsonObject(tool_calls_array_json); + if (tool_calls_array.is_array()) { + for (const auto& tool_call_obj : tool_calls_array) { + if (auto tool_call = toolCallFromJsonObject(tool_call_obj)) { + tool_calls.push_back(*tool_call); } } - - // Remove from cleaned text - size_t match_pos = offset + match.position(); - size_t match_len = match.length(); - cleaned_text.erase(match_pos, match_len); - - } catch (const json::exception& e) { - std::cerr << "[WARNING] Failed to parse [TOOL_CALLS] JSON: " << e.what() << std::endl; + eraseMatch(cleaned_text, offset + match.position() - removed, match.length()); + removed += match.length(); + } else { + std::cerr << "[WARNING] Failed to parse [TOOL_CALLS] JSON after repair attempts" << std::endl; } - + search_text = match.suffix(); offset += match.position() + match.length(); } - - // Trim whitespace from cleaned text - size_t start = cleaned_text.find_first_not_of(" \t\n\r"); - size_t end = cleaned_text.find_last_not_of(" \t\n\r"); - if (start != std::string::npos && end != std::string::npos) { - cleaned_text = cleaned_text.substr(start, end - start + 1); - } else if (start == std::string::npos) { - cleaned_text = ""; - } - + + cleaned_text = trimWhitespace(cleaned_text); + std::cout << "[ToolCalls DEBUG] Extracted " << tool_calls.size() << " tool call(s)" << std::endl; - + return {tool_calls, cleaned_text}; } json formatToolCallsForOpenAI(const std::vector& tool_calls) { json openai_tool_calls = json::array(); - + int index = 0; for (const auto& tool_call : tool_calls) { - // Generate a unique ID for each tool call std::string tool_call_id = "call_" + std::to_string(std::time(nullptr)) + "_" + std::to_string(index++); - + json openai_tool_call = { {"id", tool_call_id}, {"type", "function"}, {"function", { {"name", tool_call.name}, - {"arguments", tool_call.arguments.dump()} + {"arguments", toolCallArgumentsString(tool_call.arguments)} }} }; openai_tool_calls.push_back(openai_tool_call); } - + return openai_tool_calls; } -} // namespace ryzenai +std::vector buildToolCallStreamChunks(const json& openai_tool_calls, + const std::string& model_id) { + std::vector chunks; + if (!openai_tool_calls.is_array() || openai_tool_calls.empty()) { + return chunks; + } + + const std::time_t created = std::time(nullptr); + const std::string completion_id = "chatcmpl-" + std::to_string(created); + + for (size_t i = 0; i < openai_tool_calls.size(); ++i) { + const json& tool_call = openai_tool_calls[i]; + if (!tool_call.is_object() || !tool_call.contains("function")) { + continue; + } + + json chunk = { + {"id", completion_id}, + {"object", "chat.completion.chunk"}, + {"created", created}, + {"model", model_id}, + {"choices", json::array({ + { + {"index", 0}, + {"delta", { + {"tool_calls", json::array({ + { + {"index", static_cast(i)}, + {"id", tool_call.value("id", "")}, + {"type", tool_call.value("type", "function")}, + {"function", tool_call["function"]} + } + })} + }}, + {"finish_reason", nullptr} + } + })} + }; + chunks.push_back(chunk.dump()); + } + + return chunks; +} +} // namespace ryzenai diff --git a/tools/test_tool_calls_repair.cpp b/tools/test_tool_calls_repair.cpp new file mode 100644 index 0000000..ea2078d --- /dev/null +++ b/tools/test_tool_calls_repair.cpp @@ -0,0 +1,141 @@ +// Standalone regression test for tool call JSON repair. +// Build: c++ -std=c++17 -I../include -I../external/json tools/test_tool_calls_repair.cpp src/tool_calls.cpp -o test_tool_calls_repair +#include "ryzenai/tool_calls.h" +#include +#include +#include + +using json = nlohmann::json; + +int main() { + // Reproduce colleague report: literal newlines in file_text + Windows path backslashes + missing closing brace. + const std::string text = + "I'll create the file.\n" + "\n" + "{\"name\": \"create\", \"arguments\": {\"file_text\": \"#include \n" + "\n" + "int main() {\n" + " std::cout << \\\"Hello, World!\\\" << std::endl;\n" + " return 0;\n" + "}\n" + "\", \"path\": \"C:\\Work\\copilot\\hello_world.cpp\"}\n" + "\n"; + + auto [tool_calls, cleaned] = ryzenai::extractToolCalls(text); + + if (tool_calls.size() != 1) { + std::cerr << "Expected 1 tool call, got " << tool_calls.size() << std::endl; + return 1; + } + + if (tool_calls[0].name != "create") { + std::cerr << "Unexpected tool name: " << tool_calls[0].name << std::endl; + return 1; + } + + if (!tool_calls[0].arguments.contains("path") || + tool_calls[0].arguments["path"] != "C:\\Work\\copilot\\hello_world.cpp") { + std::cerr << "Unexpected path argument: " << tool_calls[0].arguments.dump() << std::endl; + return 1; + } + + if (!tool_calls[0].arguments.contains("file_text")) { + std::cerr << "Missing file_text argument" << std::endl; + return 1; + } + + const std::string file_text = tool_calls[0].arguments["file_text"]; + if (file_text.find("#include ") == std::string::npos || + file_text.find("Hello, World!") == std::string::npos) { + std::cerr << "Unexpected file_text: " << file_text << std::endl; + return 1; + } + + // Multiple tool calls: cleaned text must keep prose between/after markup. + const std::string multi_tool_text = + "Sure. {\"name\":\"a\",\"arguments\":{}} and then " + "{\"name\":\"b\",\"arguments\":{}} Done."; + auto [multi_calls, multi_cleaned] = ryzenai::extractToolCalls(multi_tool_text); + if (multi_calls.size() != 2 || + multi_calls[0].name != "a" || + multi_calls[1].name != "b") { + std::cerr << "Failed multi tool call extraction" << std::endl; + return 1; + } + if (multi_cleaned.find(" and then ") == std::string::npos || + multi_cleaned.find(" Done.") == std::string::npos || + multi_cleaned.find("") != std::string::npos) { + std::cerr << "Unexpected cleaned text for multi tool calls: " << multi_cleaned << std::endl; + return 1; + } + + // Windows paths with \users-style segments must survive sanitization. + const std::string users_path_text = + "Here you go\n" + "\n" + "{\"name\": \"read_file\", \"arguments\": {\"path\": \"C:\\users\\me\\data.txt\"}}\n" + "\n"; + auto [users_calls, users_cleaned] = ryzenai::extractToolCalls(users_path_text); + if (users_calls.size() != 1 || + users_calls[0].name != "read_file" || + !users_calls[0].arguments.contains("path") || + users_calls[0].arguments["path"] != "C:\\users\\me\\data.txt") { + std::cerr << "Failed \\users path tool call extraction: " + << users_calls.size() << " calls, args=" + << (users_calls.empty() ? "none" : users_calls[0].arguments.dump()) + << std::endl; + return 1; + } + + // Truncated tool call without closing tag. + const std::string truncated = + "Creating file now\n" + "\n" + "{\"name\": \"create\", \"arguments\": {\"file_text\": \"print('hi')\", \"path\": \"C:\\tmp\\a.py\"}"; + + auto [trunc_calls, trunc_cleaned] = ryzenai::extractToolCalls(truncated); + if (trunc_calls.size() != 1 || trunc_calls[0].name != "create") { + std::cerr << "Failed truncated tool call extraction" << std::endl; + return 1; + } + + // arguments provided as a JSON string (common with some models) + const std::string string_args_text = + "Creating file\n" + "\n" + "{\"name\": \"create\", \"arguments\": \"{\\\"path\\\": \\\"C:\\\\tmp\\\\a.py\\\", \\\"file_text\\\": \\\"print('hi')\\\"}\"}\n" + "\n"; + + auto [string_arg_calls, string_arg_cleaned] = ryzenai::extractToolCalls(string_args_text); + if (string_arg_calls.size() != 1 || + !string_arg_calls[0].arguments.is_object() || + string_arg_calls[0].arguments["path"] != "C:\\tmp\\a.py") { + std::cerr << "Failed string-arguments tool call extraction" << std::endl; + return 1; + } + + json openai_tool_calls = ryzenai::formatToolCallsForOpenAI(string_arg_calls); + if (!openai_tool_calls.is_array() || openai_tool_calls.size() != 1) { + std::cerr << "Unexpected OpenAI tool call formatting" << std::endl; + return 1; + } + if (!openai_tool_calls[0]["function"]["arguments"].is_string()) { + std::cerr << "OpenAI arguments must be a JSON string" << std::endl; + return 1; + } + + auto stream_chunks = ryzenai::buildToolCallStreamChunks(openai_tool_calls, "test-model"); + if (stream_chunks.size() != 1) { + std::cerr << "Expected one stream chunk" << std::endl; + return 1; + } + json stream_chunk = json::parse(stream_chunks[0]); + if (stream_chunk["choices"][0]["delta"]["tool_calls"][0]["index"] != 0 || + stream_chunk["choices"][0]["delta"]["tool_calls"][0]["function"]["name"] != "create") { + std::cerr << "Unexpected stream chunk payload: " << stream_chunks[0] << std::endl; + return 1; + } + + std::cout << "tool call repair tests passed" << std::endl; + return 0; +}