From 74cc07a2132a84075a26df67e3384776401f18f7 Mon Sep 17 00:00:00 2001 From: shili9 Date: Tue, 28 Jul 2026 05:54:33 -0500 Subject: [PATCH 1/6] tmp save 1 for kvcache reuse --- include/ryzenai/inference_engine.h | 42 +++- include/ryzenai/types.h | 1 + src/inference_engine.cpp | 340 ++++++++++++++++++++++++++++- src/server.cpp | 33 ++- src/types.cpp | 4 + 5 files changed, 407 insertions(+), 13 deletions(-) diff --git a/include/ryzenai/inference_engine.h b/include/ryzenai/inference_engine.h index ca55da9..e2598e7 100644 --- a/include/ryzenai/inference_engine.h +++ b/include/ryzenai/inference_engine.h @@ -5,6 +5,7 @@ #include #include #include +#include // Forward declarations for ONNX Runtime GenAI struct OgaModel; @@ -16,6 +17,9 @@ struct OgaMultiModalProcessor; namespace ryzenai { +// Single global multi-turn session until clients opt in via conversation_id. +inline constexpr const char* kDefaultConversationId = "default"; + // Timing data returned from completion struct CompletionTimingData { int token_count = 0; // Number of generated tokens @@ -41,7 +45,23 @@ class InferenceEngine { StreamCallback callback); // Apply chat template to messages - std::string applyChatTemplate(const std::string& messages_json, const std::string& tools_json = ""); + std::string applyChatTemplate(const std::string& messages_json, + const std::string& tools_json = "", + bool add_generation_prompt = true); + + // Multi-turn chat (non-tool): reuse one OgaGenerator per conversation_id and + // AppendTokens only the delta prompt on subsequent turns. + std::string completeMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + CompletionTimingData* out_timing = nullptr); + + void streamMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + StreamCallback callback); + + void resetMultiTurnSession(const std::string& conversation_id); // Apply the model's chat template strictly via OGA (jinja), without the // text-only manual fallbacks. Required for multimodal models whose template @@ -85,6 +105,26 @@ class InferenceEngine { std::string resolveModelPath(const std::string& path); std::vector truncatePrompt(const std::vector& input_ids); bool validateModelDirectory(const std::string& path); + + struct ChatSession { + std::unique_ptr generator; + std::unique_ptr gen_params; + std::string cached_prefix; + size_t turn_count = 0; + }; + + ChatSession& getOrCreateChatSession(const std::string& conversation_id); + std::string extractDeltaPrompt(const std::string& full_prompt, const ChatSession& session) const; + void appendPromptText(OgaGenerator& generator, const std::string& text); + void configureGeneratorParams(OgaGeneratorParams& gen_params, + const GenerationParams& params, + int total_max_length) const; + std::string applyStopSequences(const std::string& text, const GenerationParams& params) const; + std::string updateCachedPrefixAfterTurn(const std::string& messages_json, + const std::string& assistant_output); + std::vector encodeText(const std::string& text) const; + + std::unordered_map chat_sessions_; std::unique_ptr model_; std::unique_ptr tokenizer_; diff --git a/include/ryzenai/types.h b/include/ryzenai/types.h index 55b840c..b93b0d2 100644 --- a/include/ryzenai/types.h +++ b/include/ryzenai/types.h @@ -43,6 +43,7 @@ struct CompletionRequest { // Chat completion request (OpenAI format) struct ChatCompletionRequest { std::vector messages; + std::string conversation_id; // reserved: parsed but not required; server uses default session int max_tokens = 1500; float temperature = 0.7f; float top_p = 0.9f; diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index b76ac35..97c6e3a 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -8,11 +8,55 @@ #include #include #include +#include +#include namespace ryzenai { namespace fs = std::filesystem; +namespace { + +void writeAppendDebugFile(const std::string& label, + const std::string& session_id, + size_t turn_number, + const std::string& text) { + const char* debug_dir_env = std::getenv("AMDGPU_DEBUG_DIR"); + if (!debug_dir_env || debug_dir_env[0] == '\0') { + return; + } + + try { + fs::path dir(debug_dir_env); + fs::create_directories(dir); + + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + std::time_t tt = std::chrono::system_clock::to_time_t(now); + std::tm local_tm{}; + localtime_s(&local_tm, &tt); + + std::ostringstream name; + name << "server_append_" << std::put_time(&local_tm, "%Y%m%d_%H%M%S") + << '_' << std::setw(3) << std::setfill('0') << ms.count() + << ".txt"; + + fs::path out = dir / name.str(); + std::ofstream file(out); + file << "label=" << label << "\n"; + file << "session=" << session_id << "\n"; + file << "turn=" << turn_number << "\n"; + file << "chars=" << text.length() << "\n"; + file << "----text----\n"; + file << text; + std::cout << "[InferenceEngine] Wrote append debug: " << out.string() << std::endl; + } catch (const std::exception& e) { + std::cerr << "[WARNING] Failed to write append debug file: " << e.what() << std::endl; + } +} + +} // namespace + InferenceEngine::InferenceEngine(const std::string& model_path, int ctx_size) : ctx_size_(ctx_size) { @@ -125,7 +169,9 @@ GenerationParams InferenceEngine::getDefaultParams() const { return default_params_; } -std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, const std::string& tools_json) { +std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, + const std::string& tools_json, + bool add_generation_prompt) { // Parse messages json messages = json::parse(messages_json); std::ostringstream prompt; @@ -145,7 +191,7 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, template_str, messages_json.c_str(), tools_str, - true + add_generation_prompt ); std::cout << "[InferenceEngine] Applied chat template with tools" << std::endl; @@ -165,8 +211,9 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, << content << "<|im_end|>\n"; } - // Add generation prompt for assistant - prompt << "<|im_start|>assistant\n"; + if (add_generation_prompt) { + prompt << "<|im_start|>assistant\n"; + } std::cout << "[InferenceEngine] Applied Qwen/ChatML template" << std::endl; } else { @@ -178,7 +225,7 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, template_str, messages_json.c_str(), nullptr, - true + add_generation_prompt ); return std::string(result); @@ -202,7 +249,9 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, } } - prompt << "Assistant: "; + if (add_generation_prompt) { + prompt << "Assistant: "; + } } } @@ -702,6 +751,285 @@ int InferenceEngine::countTokens(const std::string& text) { } } +std::vector InferenceEngine::encodeText(const std::string& text) const { + auto sequences = OgaSequences::Create(); + tokenizer_->Encode(text.c_str(), *sequences); + const int32_t* input_ids_ptr = sequences->SequenceData(0); + size_t input_ids_count = sequences->SequenceCount(0); + return std::vector(input_ids_ptr, input_ids_ptr + input_ids_count); +} + +void InferenceEngine::appendPromptText(OgaGenerator& generator, const std::string& text) { + if (text.empty()) { + return; + } + std::vector input_ids = encodeText(text); + generator.AppendTokens(input_ids.data(), input_ids.size()); +} + +void InferenceEngine::configureGeneratorParams(OgaGeneratorParams& gen_params, + const GenerationParams& params, + int total_max_length) const { + gen_params.SetSearchOption("max_length", total_max_length); + gen_params.SetSearchOption("temperature", params.temperature); + gen_params.SetSearchOption("top_p", params.top_p); + gen_params.SetSearchOption("top_k", static_cast(params.top_k)); + gen_params.SetSearchOption("repetition_penalty", params.repetition_penalty); + gen_params.SetSearchOptionBool("do_sample", params.do_sample); + gen_params.SetSearchOption("random_seed", 1.0); +} + +std::string InferenceEngine::applyStopSequences(const std::string& text, + const GenerationParams& params) const { + std::string result = text; + for (const auto& stop_seq : params.stop_sequences) { + size_t pos = result.find(stop_seq); + if (pos != std::string::npos) { + result = result.substr(0, pos); + break; + } + } + return result; +} + +std::string InferenceEngine::updateCachedPrefixAfterTurn(const std::string& messages_json, + const std::string& assistant_output) { + json messages = json::parse(messages_json); + messages.push_back({{"role", "assistant"}, {"content", assistant_output}}); + return applyChatTemplate(messages.dump(), "", false); +} + +InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std::string& conversation_id) { + auto it = chat_sessions_.find(conversation_id); + if (it != chat_sessions_.end()) { + return it->second; + } + + ChatSession session; + session.gen_params = OgaGeneratorParams::Create(*model_); + session.generator = OgaGenerator::Create(*model_, *session.gen_params); + session.turn_count = 0; + session.cached_prefix.clear(); + + auto [inserted_it, inserted] = chat_sessions_.emplace(conversation_id, std::move(session)); + if (!inserted) { + throw std::runtime_error("Failed to create chat session: " + conversation_id); + } + + std::cout << "[InferenceEngine] Created multi-turn session: " << conversation_id << std::endl; + return inserted_it->second; +} + +std::string InferenceEngine::extractDeltaPrompt(const std::string& full_prompt, + const ChatSession& session) const { + if (session.cached_prefix.empty()) { + return full_prompt; + } + + if (full_prompt.size() < session.cached_prefix.size() || + full_prompt.compare(0, session.cached_prefix.size(), session.cached_prefix) != 0) { + throw std::runtime_error( + "Multi-turn prefix mismatch: full prompt does not extend session cached prefix. " + "Ensure assistant history matches prior model output, or restart the server."); + } + + return full_prompt.substr(session.cached_prefix.size()); +} + +void InferenceEngine::resetMultiTurnSession(const std::string& conversation_id) { + std::lock_guard lock(inference_mutex_); + chat_sessions_.erase(conversation_id); + std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id << std::endl; +} + +std::string InferenceEngine::completeMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + CompletionTimingData* out_timing) { + std::lock_guard lock(inference_mutex_); + + try { + const std::string full_prompt = applyChatTemplate(messages_json, "", true); + ChatSession& session = getOrCreateChatSession(conversation_id); + const bool is_first_turn = (session.turn_count == 0); + const std::string delta_prompt = extractDeltaPrompt(full_prompt, session); + + if (!is_first_turn && delta_prompt.empty()) { + throw std::runtime_error("Multi-turn request produced an empty delta prompt"); + } + + const std::string text_to_append = is_first_turn ? full_prompt : delta_prompt; + const int append_token_count = countTokens(text_to_append); + + std::cout << "[InferenceEngine] Multi-turn turn=" << (session.turn_count + 1) + << " session=" << conversation_id + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " append_chars=" << text_to_append.length() + << " append_tokens=" << append_token_count + << " full_tokens=" << countTokens(full_prompt) + << " cached_chars=" << session.cached_prefix.length() << std::endl; + if (!is_first_turn) { + std::cout << "[InferenceEngine] Multi-turn delta (first 200): " + << delta_prompt.substr(0, std::min(size_t(200), delta_prompt.length())) + << std::endl; + } + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + const size_t seq_before = session.generator->GetSequenceCount(0); + appendPromptText(*session.generator, text_to_append); + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto start_time = std::chrono::high_resolution_clock::now(); + auto first_token_time = start_time; + bool first_token_received = false; + + while (!session.generator->IsDone()) { + session.generator->GenerateNextToken(); + if (!first_token_received) { + first_token_time = std::chrono::high_resolution_clock::now(); + first_token_received = true; + } + } + + auto end_time = std::chrono::high_resolution_clock::now(); + const int32_t* output_ptr = session.generator->GetSequenceData(0); + const size_t output_count = session.generator->GetSequenceCount(0); + const int generated_token_count = (output_count > seq_before) + ? static_cast(output_count - seq_before) + : 0; + + std::string result; + if (output_count > seq_before) { + auto decoded = tokenizer_->Decode(output_ptr + seq_before, output_count - seq_before); + result = applyStopSequences(std::string(decoded), params); + } + + session.cached_prefix = updateCachedPrefixAfterTurn(messages_json, result); + session.turn_count++; + + if (out_timing != nullptr) { + auto total_duration = std::chrono::duration_cast(end_time - start_time); + auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); + const double ttft_seconds = ttft_duration.count() / 1000.0; + const double total_time_ms = static_cast(total_duration.count()); + const double decode_time_seconds = (total_duration.count() - ttft_duration.count()) / 1000.0; + double tps = 0.0; + if (generated_token_count > 1 && decode_time_seconds > 0) { + tps = (generated_token_count - 1) / decode_time_seconds; + } else if (generated_token_count == 1 && total_time_ms > 0) { + tps = 1.0 / (total_time_ms / 1000.0); + } + + out_timing->token_count = generated_token_count; + out_timing->ttft_seconds = ttft_seconds; + out_timing->tps = tps; + out_timing->total_time_ms = total_time_ms; + } + + std::cout << "[InferenceEngine] Multi-turn completed turn=" << session.turn_count + << " generated=" << generated_token_count + << " cached_prefix_len=" << session.cached_prefix.length() + << " token_count=" << session.generator->TokenCount() << std::endl; + + return result; + } catch (const std::exception& e) { + throw std::runtime_error("Multi-turn inference failed: " + std::string(e.what())); + } +} + +void InferenceEngine::streamMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + StreamCallback callback) { + std::lock_guard lock(inference_mutex_); + + try { + const std::string full_prompt = applyChatTemplate(messages_json, "", true); + ChatSession& session = getOrCreateChatSession(conversation_id); + const bool is_first_turn = (session.turn_count == 0); + const std::string delta_prompt = extractDeltaPrompt(full_prompt, session); + + if (!is_first_turn && delta_prompt.empty()) { + throw std::runtime_error("Multi-turn request produced an empty delta prompt"); + } + + const std::string text_to_append = is_first_turn ? full_prompt : delta_prompt; + + std::cout << "[InferenceEngine] Multi-turn stream turn=" << (session.turn_count + 1) + << " session=" << conversation_id + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " append_tokens=" << countTokens(text_to_append) << std::endl; + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + appendPromptText(*session.generator, text_to_append); + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer_); + std::string accumulated_output; + bool client_disconnected = false; + + while (!session.generator->IsDone() && !client_disconnected) { + session.generator->GenerateNextToken(); + + const int32_t* all_tokens = session.generator->GetSequenceData(0); + const size_t num_tokens = session.generator->GetSequenceCount(0); + const int32_t new_token = all_tokens[num_tokens - 1]; + + const char* decoded = tokenizer_stream->Decode(new_token); + if (decoded && decoded[0] != '\0') { + std::string token_str(decoded); + + bool should_stop = false; + for (const auto& stop_seq : params.stop_sequences) { + std::string temp_output = accumulated_output + token_str; + if (temp_output.find(stop_seq) != std::string::npos) { + should_stop = true; + break; + } + } + if (should_stop) { + break; + } + + accumulated_output += token_str; + const bool is_final = session.generator->IsDone(); + if (!callback(token_str, is_final)) { + client_disconnected = true; + break; + } + } + } + + accumulated_output = applyStopSequences(accumulated_output, params); + session.cached_prefix = updateCachedPrefixAfterTurn(messages_json, accumulated_output); + session.turn_count++; + + std::cout << "[InferenceEngine] Multi-turn stream completed turn=" << session.turn_count + << " cached_prefix_len=" << session.cached_prefix.length() + << " token_count=" << session.generator->TokenCount() << std::endl; + } catch (const std::exception& e) { + throw std::runtime_error("Multi-turn streaming failed: " + std::string(e.what())); + } +} + void InferenceEngine::detectMultimodal() { // A model is multimodal if genai_config.json declares a vision (or speech) // pipeline, or a processor_config.json is present alongside the model. diff --git a/src/server.cpp b/src/server.cpp index 4564a9d..9aacd65 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -643,6 +643,11 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string prompt = inference_engine_->applyChatTemplate(messages_array.dump(), tools_json); std::cout << "[Server DEBUG] Generated prompt length: " << prompt.length() << " chars" << std::endl; std::cout << "[Server DEBUG] Prompt (first 500 chars): " << prompt.substr(0, std::min(size_t(500), prompt.length())) << std::endl; + + // Text chat always uses the single default multi-turn AppendTokens session. + // (Tool definitions in the request do not disable this; only post-hoc tool parsing.) + const bool use_multi_turn = true; + const std::string session_id = kDefaultConversationId; if (chat_req.stream) { // REAL-TIME STREAMING: Send chunks as tokens are generated @@ -662,9 +667,12 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Count prompt tokens before streaming int prompt_tokens = inference_engine_->countTokens(prompt); + std::string messages_json = messages_array.dump(); + res.set_chunked_content_provider( "text/event-stream", - [this, prompt, params, model_id, has_tools, prompt_tokens](size_t offset, httplib::DataSink& sink) { + [this, prompt, params, model_id, has_tools, prompt_tokens, use_multi_turn, + session_id, messages_json](size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; // Only run once try { @@ -680,8 +688,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Create reasoning parser for streaming ReasoningStreamParser reasoning_parser; - // Generate and send tokens in real-time - inference_engine_->streamComplete(prompt, params, + StreamCallback stream_callback = [&sink, model_id, &token_count, &full_response, &reasoning_parser, &first_token_received, &first_token_time](const std::string& token, bool is_final) -> bool { // Track time to first token if (!first_token_received && !token.empty()) { @@ -793,8 +800,13 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: token_count++; return true; // Continue generation - } - ); + }; + + if (use_multi_turn) { + inference_engine_->streamMultiTurn(session_id, messages_json, params, stream_callback); + } else { + inference_engine_->streamComplete(prompt, params, stream_callback); + } // After generation completes, do a final flush to catch any remaining buffered content // This handles the case where the last few tokens didn't trigger processing due to buffer size @@ -938,7 +950,16 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: ); CompletionTimingData timing; - std::string output = inference_engine_->complete(prompt, params, &timing); + std::string output; + if (use_multi_turn) { + output = inference_engine_->completeMultiTurn( + session_id, + messages_array.dump(), + params, + &timing); + } else { + output = inference_engine_->complete(prompt, params, &timing); + } // Parse reasoning content from output auto reasoning_result = parseReasoningContent(output); diff --git a/src/types.cpp b/src/types.cpp index 0e57953..e096360 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -140,6 +140,10 @@ ChatCompletionRequest ChatCompletionRequest::fromJSON(const json& j) { if (j.contains("tools")) { req.tools = j["tools"]; } + + if (j.contains("conversation_id") && j["conversation_id"].is_string()) { + req.conversation_id = j["conversation_id"]; + } return req; } From 66a598a1ad0e7c58d25d5846cab9ab5901a3049f Mon Sep 17 00:00:00 2001 From: shili9 Date: Tue, 28 Jul 2026 06:52:03 -0500 Subject: [PATCH 2/6] 1st version work: only for llm append --- include/ryzenai/inference_engine.h | 6 ++-- src/inference_engine.cpp | 56 +++++++++++------------------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/include/ryzenai/inference_engine.h b/include/ryzenai/inference_engine.h index e2598e7..8ac72e6 100644 --- a/include/ryzenai/inference_engine.h +++ b/include/ryzenai/inference_engine.h @@ -109,19 +109,17 @@ class InferenceEngine { struct ChatSession { std::unique_ptr generator; std::unique_ptr gen_params; - std::string cached_prefix; size_t turn_count = 0; }; ChatSession& getOrCreateChatSession(const std::string& conversation_id); - std::string extractDeltaPrompt(const std::string& full_prompt, const ChatSession& session) const; + // Extract the latest user turn from full prompt: suffix starting at last <|im_start|>user. + std::string extractDeltaPromptFromLastUser(const std::string& full_prompt) const; void appendPromptText(OgaGenerator& generator, const std::string& text); void configureGeneratorParams(OgaGeneratorParams& gen_params, const GenerationParams& params, int total_max_length) const; std::string applyStopSequences(const std::string& text, const GenerationParams& params) const; - std::string updateCachedPrefixAfterTurn(const std::string& messages_json, - const std::string& assistant_output); std::vector encodeText(const std::string& text) const; std::unordered_map chat_sessions_; diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index 97c6e3a..7eac592 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -792,11 +792,15 @@ std::string InferenceEngine::applyStopSequences(const std::string& text, return result; } -std::string InferenceEngine::updateCachedPrefixAfterTurn(const std::string& messages_json, - const std::string& assistant_output) { - json messages = json::parse(messages_json); - messages.push_back({{"role", "assistant"}, {"content", assistant_output}}); - return applyChatTemplate(messages.dump(), "", false); +std::string InferenceEngine::extractDeltaPromptFromLastUser(const std::string& full_prompt) const { + static constexpr const char* kUserTurnMarker = "<|im_start|>user"; + const size_t pos = full_prompt.rfind(kUserTurnMarker); + if (pos == std::string::npos) { + throw std::runtime_error( + std::string("Multi-turn: could not find latest user turn marker '") + kUserTurnMarker + + "' in full prompt"); + } + return full_prompt.substr(pos); } InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std::string& conversation_id) { @@ -809,7 +813,6 @@ InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std: session.gen_params = OgaGeneratorParams::Create(*model_); session.generator = OgaGenerator::Create(*model_, *session.gen_params); session.turn_count = 0; - session.cached_prefix.clear(); auto [inserted_it, inserted] = chat_sessions_.emplace(conversation_id, std::move(session)); if (!inserted) { @@ -820,22 +823,6 @@ InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std: return inserted_it->second; } -std::string InferenceEngine::extractDeltaPrompt(const std::string& full_prompt, - const ChatSession& session) const { - if (session.cached_prefix.empty()) { - return full_prompt; - } - - if (full_prompt.size() < session.cached_prefix.size() || - full_prompt.compare(0, session.cached_prefix.size(), session.cached_prefix) != 0) { - throw std::runtime_error( - "Multi-turn prefix mismatch: full prompt does not extend session cached prefix. " - "Ensure assistant history matches prior model output, or restart the server."); - } - - return full_prompt.substr(session.cached_prefix.size()); -} - void InferenceEngine::resetMultiTurnSession(const std::string& conversation_id) { std::lock_guard lock(inference_mutex_); chat_sessions_.erase(conversation_id); @@ -852,13 +839,15 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i const std::string full_prompt = applyChatTemplate(messages_json, "", true); ChatSession& session = getOrCreateChatSession(conversation_id); const bool is_first_turn = (session.turn_count == 0); - const std::string delta_prompt = extractDeltaPrompt(full_prompt, session); + const std::string delta_prompt = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + const std::string text_to_append = delta_prompt; - if (!is_first_turn && delta_prompt.empty()) { + if (!is_first_turn && text_to_append.empty()) { throw std::runtime_error("Multi-turn request produced an empty delta prompt"); } - const std::string text_to_append = is_first_turn ? full_prompt : delta_prompt; const int append_token_count = countTokens(text_to_append); std::cout << "[InferenceEngine] Multi-turn turn=" << (session.turn_count + 1) @@ -866,11 +855,10 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") << " append_chars=" << text_to_append.length() << " append_tokens=" << append_token_count - << " full_tokens=" << countTokens(full_prompt) - << " cached_chars=" << session.cached_prefix.length() << std::endl; + << " full_tokens=" << countTokens(full_prompt) << std::endl; if (!is_first_turn) { std::cout << "[InferenceEngine] Multi-turn delta (first 200): " - << delta_prompt.substr(0, std::min(size_t(200), delta_prompt.length())) + << text_to_append.substr(0, std::min(size_t(200), text_to_append.length())) << std::endl; } @@ -913,7 +901,6 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i result = applyStopSequences(std::string(decoded), params); } - session.cached_prefix = updateCachedPrefixAfterTurn(messages_json, result); session.turn_count++; if (out_timing != nullptr) { @@ -937,7 +924,6 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i std::cout << "[InferenceEngine] Multi-turn completed turn=" << session.turn_count << " generated=" << generated_token_count - << " cached_prefix_len=" << session.cached_prefix.length() << " token_count=" << session.generator->TokenCount() << std::endl; return result; @@ -956,14 +942,14 @@ void InferenceEngine::streamMultiTurn(const std::string& conversation_id, const std::string full_prompt = applyChatTemplate(messages_json, "", true); ChatSession& session = getOrCreateChatSession(conversation_id); const bool is_first_turn = (session.turn_count == 0); - const std::string delta_prompt = extractDeltaPrompt(full_prompt, session); + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); - if (!is_first_turn && delta_prompt.empty()) { + if (!is_first_turn && text_to_append.empty()) { throw std::runtime_error("Multi-turn request produced an empty delta prompt"); } - const std::string text_to_append = is_first_turn ? full_prompt : delta_prompt; - std::cout << "[InferenceEngine] Multi-turn stream turn=" << (session.turn_count + 1) << " session=" << conversation_id << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") @@ -1019,11 +1005,9 @@ void InferenceEngine::streamMultiTurn(const std::string& conversation_id, } accumulated_output = applyStopSequences(accumulated_output, params); - session.cached_prefix = updateCachedPrefixAfterTurn(messages_json, accumulated_output); session.turn_count++; std::cout << "[InferenceEngine] Multi-turn stream completed turn=" << session.turn_count - << " cached_prefix_len=" << session.cached_prefix.length() << " token_count=" << session.generator->TokenCount() << std::endl; } catch (const std::exception& e) { throw std::runtime_error("Multi-turn streaming failed: " + std::string(e.what())); From ad553a6c7aea60b7a1c1d7a05c63f063946c8350 Mon Sep 17 00:00:00 2001 From: shili9 Date: Tue, 28 Jul 2026 07:37:38 -0500 Subject: [PATCH 3/6] 2st version work: reset OK by rewindTo(0) --- include/ryzenai/server.h | 3 ++- src/inference_engine.cpp | 20 ++++++++++++++++++-- src/server.cpp | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/include/ryzenai/server.h b/include/ryzenai/server.h index 9f27a7a..3428035 100644 --- a/include/ryzenai/server.h +++ b/include/ryzenai/server.h @@ -35,7 +35,8 @@ class RyzenAIServer { void handleMultimodalChat(const json& request_json, const ChatCompletionRequest& chat_req, const std::vector& images, httplib::Response& res); void handleResponses(const httplib::Request& req, httplib::Response& res); - + void handleSessionReset(const httplib::Request& req, httplib::Response& res); + // Helper methods json createErrorResponse(const std::string& message, const std::string& type); void setupCORS(httplib::Response& res); diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index 7eac592..8aecdb9 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -825,8 +825,24 @@ InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std: void InferenceEngine::resetMultiTurnSession(const std::string& conversation_id) { std::lock_guard lock(inference_mutex_); - chat_sessions_.erase(conversation_id); - std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id << std::endl; + auto it = chat_sessions_.find(conversation_id); + if (it == chat_sessions_.end()) { + std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id + << " (no existing session, no-op)" << std::endl; + return; + } + + ChatSession& session = it->second; + try { + session.generator->RewindTo(0); + session.turn_count = 0; + std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id + << " (RewindTo(0), turn_count=0)" << std::endl; + } catch (const std::exception& e) { + std::cerr << "[InferenceEngine] RewindTo(0) failed for session " << conversation_id + << ", recreating generator: " << e.what() << std::endl; + chat_sessions_.erase(it); + } } std::string InferenceEngine::completeMultiTurn(const std::string& conversation_id, diff --git a/src/server.cpp b/src/server.cpp index 9aacd65..374c69b 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -221,6 +221,10 @@ void RyzenAIServer::setupRoutes() { http_server_->Post("/v1/responses", [this](const httplib::Request& req, httplib::Response& res) { handleResponses(req, res); }); + + http_server_->Post("/v1/sessions/reset", [this](const httplib::Request& req, httplib::Response& res) { + handleSessionReset(req, res); + }); // Root redirect http_server_->Get("/", [this](const httplib::Request&, httplib::Response& res) { @@ -232,7 +236,8 @@ void RyzenAIServer::setupRoutes() { "/health", "/v1/completions", "/v1/chat/completions", - "/v1/responses" + "/v1/responses", + "/v1/sessions/reset" }} }; res.set_content(response.dump(2), "application/json"); @@ -263,6 +268,32 @@ void RyzenAIServer::handleHealth(const httplib::Request& req, httplib::Response& res.set_content(response.dump(2), "application/json"); } +void RyzenAIServer::handleSessionReset(const httplib::Request& req, httplib::Response& res) { + try { + std::string conversation_id = kDefaultConversationId; + if (!req.body.empty()) { + json request_json = json::parse(req.body); + if (request_json.contains("conversation_id") && + request_json["conversation_id"].is_string()) { + conversation_id = request_json["conversation_id"].get(); + } + } + + inference_engine_->resetMultiTurnSession(conversation_id); + + json response = { + {"status", "success"}, + {"conversation_id", conversation_id}, + {"message", "Chat session KV cache reset"} + }; + res.set_content(response.dump(), "application/json"); + } catch (const std::exception& e) { + res.status = 500; + res.set_content(createErrorResponse(e.what(), "session_reset_error").dump(), + "application/json"); + } +} + void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Response& res) { try { // Parse request @@ -1176,6 +1207,7 @@ void RyzenAIServer::run() { std::cout << " GET http://" << args_.host << ":" << args_.port << "/health\n"; std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/completions\n"; std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/chat/completions\n"; + std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/sessions/reset\n"; std::cout << "\n"; std::cout << "Press Ctrl+C to stop the server\n"; std::cout << "===============================================================\n\n"; From 45dc1d3b3cd418b01011fb58bad9e231f2e4ba90 Mon Sep 17 00:00:00 2001 From: shili9 Date: Tue, 28 Jul 2026 08:49:26 -0500 Subject: [PATCH 4/6] 1st with pic, 2nd only text, kvcache reuse OK --- include/ryzenai/inference_engine.h | 19 +++ include/ryzenai/server.h | 5 +- src/inference_engine.cpp | 256 +++++++++++++++++++++++++++++ src/server.cpp | 95 ++++++++--- 4 files changed, 350 insertions(+), 25 deletions(-) diff --git a/include/ryzenai/inference_engine.h b/include/ryzenai/inference_engine.h index 8ac72e6..a537e0b 100644 --- a/include/ryzenai/inference_engine.h +++ b/include/ryzenai/inference_engine.h @@ -63,6 +63,24 @@ class InferenceEngine { void resetMultiTurnSession(const std::string& conversation_id); + // Multi-turn VLM: reuse ChatSession KV cache; turn 1 may SetInputs (vision), + // later text-only turns append delta from last <|im_start|>user via jinja template. + std::string completeMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + CompletionTimingData* out_timing = nullptr); + + void streamMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + StreamCallback callback); + // Apply the model's chat template strictly via OGA (jinja), without the // text-only manual fallbacks. Required for multimodal models whose template // expands image placeholders (e.g. <|vision_start|><|image_pad|><|vision_end|>). @@ -113,6 +131,7 @@ class InferenceEngine { }; ChatSession& getOrCreateChatSession(const std::string& conversation_id); + void resetMultiTurnSessionLocked(const std::string& conversation_id); // Extract the latest user turn from full prompt: suffix starting at last <|im_start|>user. std::string extractDeltaPromptFromLastUser(const std::string& full_prompt) const; void appendPromptText(OgaGenerator& generator, const std::string& text); diff --git a/include/ryzenai/server.h b/include/ryzenai/server.h index 3428035..a2dc0e6 100644 --- a/include/ryzenai/server.h +++ b/include/ryzenai/server.h @@ -33,7 +33,10 @@ class RyzenAIServer { void handleCompletions(const httplib::Request& req, httplib::Response& res); void handleChatCompletions(const httplib::Request& req, httplib::Response& res); void handleMultimodalChat(const json& request_json, const ChatCompletionRequest& chat_req, - const std::vector& images, httplib::Response& res); + const json& messages_array, + const std::vector& new_turn_images, + const std::vector& all_images, + httplib::Response& res); void handleResponses(const httplib::Request& req, httplib::Response& res); void handleSessionReset(const httplib::Request& req, httplib::Response& res); diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index 8aecdb9..534f645 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -825,6 +825,10 @@ InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std: void InferenceEngine::resetMultiTurnSession(const std::string& conversation_id) { std::lock_guard lock(inference_mutex_); + resetMultiTurnSessionLocked(conversation_id); +} + +void InferenceEngine::resetMultiTurnSessionLocked(const std::string& conversation_id) { auto it = chat_sessions_.find(conversation_id); if (it == chat_sessions_.end()) { std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id @@ -1089,6 +1093,258 @@ void apply_search_options(OgaGeneratorParams& gen_params, const GenerationParams } } // namespace +namespace { + +std::unique_ptr loadOgaImages(const std::vector& images) { + if (images.empty()) { + return nullptr; + } + std::vector data_ptrs; + std::vector data_sizes; + data_ptrs.reserve(images.size()); + data_sizes.reserve(images.size()); + for (const auto& img : images) { + data_ptrs.push_back(img.data()); + data_sizes.push_back(img.size()); + } + return OgaImages::Load(data_ptrs.data(), data_sizes.data(), data_ptrs.size()); +} + +} // namespace + +std::string InferenceEngine::completeMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + CompletionTimingData* out_timing) { + std::lock_guard lock(inference_mutex_); + if (!processor_) { + throw std::runtime_error("Multimodal multi-turn requested but model is not multimodal"); + } + + try { + const std::string full_prompt = applyChatTemplateRaw(messages_json, tools_json); + bool is_first_turn = true; + if (auto it = chat_sessions_.find(conversation_id); it != chat_sessions_.end()) { + is_first_turn = (it->second.turn_count == 0); + } + + if (!is_first_turn && !new_turn_images.empty()) { + std::cout << "[InferenceEngine] Multimodal multi-turn: new images on follow-up turn, " + << "resetting session and reprocessing full prompt" << std::endl; + chat_sessions_.erase(conversation_id); + is_first_turn = true; + } + + ChatSession& active_session = getOrCreateChatSession(conversation_id); + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + + if (text_to_append.empty()) { + throw std::runtime_error("Multimodal multi-turn request produced an empty prompt segment"); + } + + const std::vector& images_for_process = + is_first_turn ? all_images : new_turn_images; + + std::cout << "[InferenceEngine] Multimodal multi-turn turn=" << (active_session.turn_count + 1) + << " session=" << conversation_id + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " append_chars=" << text_to_append.length() + << " append_tokens=" << countTokens(text_to_append) + << " new_turn_images=" << new_turn_images.size() + << " process_images=" << images_for_process.size() + << " full_tokens=" << countTokens(full_prompt) << std::endl; + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + active_session.turn_count + 1, + text_to_append); + + const size_t seq_before = active_session.generator->GetSequenceCount(0); + + if (is_first_turn && !images_for_process.empty()) { + auto oga_images = loadOgaImages(images_for_process); + auto inputs = processor_->ProcessImages(text_to_append.c_str(), oga_images.get()); + active_session.generator->SetInputs(*inputs); + } else { + appendPromptText(*active_session.generator, text_to_append); + } + + int total_max_length = static_cast(active_session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*active_session.gen_params, params, total_max_length); + + auto start_time = std::chrono::high_resolution_clock::now(); + auto first_token_time = start_time; + bool first_token_received = false; + + while (!active_session.generator->IsDone()) { + active_session.generator->GenerateNextToken(); + if (!first_token_received) { + first_token_time = std::chrono::high_resolution_clock::now(); + first_token_received = true; + } + } + + auto end_time = std::chrono::high_resolution_clock::now(); + const int32_t* output_ptr = active_session.generator->GetSequenceData(0); + const size_t output_count = active_session.generator->GetSequenceCount(0); + const int generated_token_count = (output_count > seq_before) + ? static_cast(output_count - seq_before) + : 0; + + std::string result; + if (output_count > seq_before) { + auto decoded = processor_->Decode(output_ptr + seq_before, output_count - seq_before); + result = applyStopSequences(std::string(decoded), params); + } + + active_session.turn_count++; + + if (out_timing != nullptr) { + auto total_duration = std::chrono::duration_cast(end_time - start_time); + auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); + const double ttft_seconds = ttft_duration.count() / 1000.0; + const double total_time_ms = static_cast(total_duration.count()); + const double decode_time_seconds = (total_duration.count() - ttft_duration.count()) / 1000.0; + double tps = 0.0; + if (generated_token_count > 1 && decode_time_seconds > 0) { + tps = (generated_token_count - 1) / decode_time_seconds; + } else if (generated_token_count == 1 && total_time_ms > 0) { + tps = 1.0 / (total_time_ms / 1000.0); + } + + out_timing->token_count = generated_token_count; + out_timing->ttft_seconds = ttft_seconds; + out_timing->tps = tps; + out_timing->total_time_ms = total_time_ms; + } + + std::cout << "[InferenceEngine] Multimodal multi-turn completed turn=" << active_session.turn_count + << " generated=" << generated_token_count << std::endl; + return result; + } catch (const std::exception& e) { + throw std::runtime_error("Multimodal multi-turn inference failed: " + std::string(e.what())); + } +} + +void InferenceEngine::streamMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + StreamCallback callback) { + std::lock_guard lock(inference_mutex_); + if (!processor_) { + throw std::runtime_error("Multimodal multi-turn requested but model is not multimodal"); + } + + try { + const std::string full_prompt = applyChatTemplateRaw(messages_json, tools_json); + bool is_first_turn = true; + if (auto it = chat_sessions_.find(conversation_id); it != chat_sessions_.end()) { + is_first_turn = (it->second.turn_count == 0); + } + + if (!is_first_turn && !new_turn_images.empty()) { + std::cout << "[InferenceEngine] Multimodal multi-turn stream: new images on follow-up turn, " + << "resetting session and reprocessing full prompt" << std::endl; + chat_sessions_.erase(conversation_id); + is_first_turn = true; + } + + ChatSession& session = getOrCreateChatSession(conversation_id); + + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + + if (text_to_append.empty()) { + throw std::runtime_error("Multimodal multi-turn request produced an empty prompt segment"); + } + + const std::vector& images_for_process = + is_first_turn ? all_images : new_turn_images; + + std::cout << "[InferenceEngine] Multimodal multi-turn stream turn=" << (session.turn_count + 1) + << " session=" << conversation_id + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " append_tokens=" << countTokens(text_to_append) + << " new_turn_images=" << new_turn_images.size() + << " process_images=" << images_for_process.size() << std::endl; + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + if (is_first_turn && !images_for_process.empty()) { + auto oga_images = loadOgaImages(images_for_process); + auto inputs = processor_->ProcessImages(text_to_append.c_str(), oga_images.get()); + session.generator->SetInputs(*inputs); + } else { + appendPromptText(*session.generator, text_to_append); + } + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto tokenizer_stream = OgaTokenizerStream::Create(*processor_); + std::string accumulated_output; + bool client_disconnected = false; + + while (!session.generator->IsDone() && !client_disconnected) { + session.generator->GenerateNextToken(); + + const int32_t* all_tokens = session.generator->GetSequenceData(0); + const size_t num_tokens = session.generator->GetSequenceCount(0); + const int32_t new_token = all_tokens[num_tokens - 1]; + + const char* decoded = tokenizer_stream->Decode(new_token); + if (decoded && decoded[0] != '\0') { + std::string token_str(decoded); + + bool should_stop = false; + for (const auto& stop_seq : params.stop_sequences) { + std::string temp_output = accumulated_output + token_str; + if (temp_output.find(stop_seq) != std::string::npos) { + should_stop = true; + break; + } + } + if (should_stop) { + break; + } + + accumulated_output += token_str; + const bool is_final = session.generator->IsDone(); + if (!callback(token_str, is_final)) { + client_disconnected = true; + break; + } + } + } + + accumulated_output = applyStopSequences(accumulated_output, params); + session.turn_count++; + + std::cout << "[InferenceEngine] Multimodal multi-turn stream completed turn=" + << session.turn_count << std::endl; + } catch (const std::exception& e) { + throw std::runtime_error("Multimodal multi-turn streaming failed: " + std::string(e.what())); + } +} + std::string InferenceEngine::completeMultimodal(const std::string& prompt, const std::vector& images, const GenerationParams& params, diff --git a/src/server.cpp b/src/server.cpp index 374c69b..f05337b 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -91,6 +91,49 @@ std::vector extract_request_images(const json& request_json) { return images; } +// Only images attached to the latest user turn. Follow-up text turns must not +// re-ingest images from earlier history (that forced full prompt reprocessing). +std::vector extract_images_from_last_user_message(const json& request_json) { + std::vector images; + if (!request_json.contains("messages") || !request_json["messages"].is_array()) { + return images; + } + + const json* last_user = nullptr; + for (const auto& msg : request_json["messages"]) { + if (msg.value("role", "") == "user") { + last_user = &msg; + } + } + if (last_user == nullptr || !last_user->contains("content") || !(*last_user)["content"].is_array()) { + return images; + } + + for (const auto& item : (*last_user)["content"]) { + if (!item.is_object()) continue; + std::string url; + if (item.contains("image_url")) { + const auto& iu = item["image_url"]; + if (iu.is_object() && iu.contains("url") && iu["url"].is_string()) { + url = iu["url"].get(); + } else if (iu.is_string()) { + url = iu.get(); + } + } else if (item.value("type", "") == "image" && item.contains("url") && item["url"].is_string()) { + url = item["url"].get(); + } + if (url.empty()) continue; + std::string bytes = resolve_image_bytes(url); + if (!bytes.empty()) { + images.push_back(std::move(bytes)); + } else { + std::cerr << "[WARNING] Could not resolve image reference in latest user turn (skipped)" + << std::endl; + } + } + return images; +} + } // namespace RyzenAIServer::RyzenAIServer(const CommandLineArgs& args) @@ -645,14 +688,14 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: return; } - // Multimodal branch: a VLM request carrying images is handled separately - // (OpenAI image_url content -> OGA vision pipeline). Returns early. + // VLM models always use the multimodal multi-turn path (KV cache reuse on + // text-only follow-ups). Only images in the latest user turn are processed. if (inference_engine_->isMultimodal()) { - std::vector images = extract_request_images(request_json); - if (!images.empty()) { - handleMultimodalChat(request_json, chat_req, images, res); - return; - } + json messages_array = request_json.contains("messages") ? request_json["messages"] : json::array(); + std::vector new_turn_images = extract_images_from_last_user_message(request_json); + std::vector all_images = extract_request_images(request_json); + handleMultimodalChat(request_json, chat_req, messages_array, new_turn_images, all_images, res); + return; } // Convert messages to JSON array for chat template. Use the raw request @@ -1081,18 +1124,17 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: void RyzenAIServer::handleMultimodalChat(const json& request_json, const ChatCompletionRequest& chat_req, - const std::vector& images, + const json& messages_array, + const std::vector& new_turn_images, + const std::vector& all_images, httplib::Response& res) { - // Build the prompt strictly via the OGA jinja template so image placeholders - // (<|vision_start|><|image_pad|><|vision_end|>) are emitted for each image. - json messages_array = request_json.contains("messages") ? request_json["messages"] : json::array(); std::string tools_json = chat_req.tools.empty() ? "" : chat_req.tools.dump(); - std::string prompt = inference_engine_->applyChatTemplateRaw(messages_array.dump(), tools_json); + const std::string session_id = kDefaultConversationId; + const std::string messages_json = messages_array.dump(); - std::cout << "[Server] Multimodal chat request (images=" << images.size() - << ", stream=" << chat_req.stream << ")" << std::endl; - std::cout << "[Server DEBUG] MM prompt (first 300): " - << prompt.substr(0, std::min(size_t(300), prompt.length())) << std::endl; + std::cout << "[Server] Multimodal chat request (new_turn_images=" << new_turn_images.size() + << ", all_images=" << all_images.size() + << ", stream=" << chat_req.stream << ", multi_turn=1)" << std::endl; GenerationParams params = createGenerationParams( chat_req.max_tokens, chat_req.temperature, chat_req.top_p, @@ -1122,15 +1164,18 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, res.set_header("Connection", "keep-alive"); res.set_header("X-Accel-Buffering", "no"); - int prompt_tokens = inference_engine_->countTokens(prompt); + std::string preview_prompt = inference_engine_->applyChatTemplateRaw(messages_json, tools_json); + int prompt_tokens = inference_engine_->countTokens(preview_prompt); res.set_chunked_content_provider( "text/event-stream", - [this, prompt, images, params, model_id, prompt_tokens, escapeJson](size_t offset, httplib::DataSink& sink) { + [this, messages_json, new_turn_images, all_images, tools_json, params, model_id, prompt_tokens, session_id, escapeJson]( + size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; try { int token_count = 0; - inference_engine_->streamCompleteMultimodal(prompt, images, params, + inference_engine_->streamMultiTurnMultimodal( + session_id, messages_json, new_turn_images, all_images, tools_json, params, [&sink, model_id, &token_count, &escapeJson](const std::string& token, bool /*is_final*/) -> bool { std::string chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + @@ -1154,9 +1199,10 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, const char* done = "data: [DONE]\n\n"; sink.write(done, strlen(done)); sink.done(); - std::cout << "[Server] [OK] Streamed " << token_count << " multimodal tokens" << std::endl; + std::cout << "[Server] [OK] Streamed " << token_count << " multimodal multi-turn tokens" + << std::endl; } catch (const std::exception& e) { - std::cerr << "[ERROR] Multimodal streaming failed: " << e.what() << std::endl; + std::cerr << "[ERROR] Multimodal multi-turn streaming failed: " << e.what() << std::endl; json err = createErrorResponse(e.what(), "inference_error"); std::string es = "data: " + err.dump() + "\n\n"; sink.write(es.c_str(), es.size()); @@ -1167,10 +1213,11 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, return; } - // Non-streaming CompletionTimingData timing; - int prompt_tokens = inference_engine_->countTokens(prompt); - std::string content = inference_engine_->completeMultimodal(prompt, images, params, &timing); + std::string preview_prompt = inference_engine_->applyChatTemplateRaw(messages_json, tools_json); + int prompt_tokens = inference_engine_->countTokens(preview_prompt); + std::string content = inference_engine_->completeMultiTurnMultimodal( + session_id, messages_json, new_turn_images, all_images, tools_json, params, &timing); json response = { {"id", "chatcmpl-" + std::to_string(std::time(nullptr))}, From 001b5b4a750fbd7778534dc10680b8271b3046f3 Mon Sep 17 00:00:00 2001 From: shili9 Date: Tue, 28 Jul 2026 09:23:43 -0500 Subject: [PATCH 5/6] debug code for copilot --- src/inference_engine.cpp | 174 ++++++++++++++++++++++++++++++++------- src/server.cpp | 24 +++++- 2 files changed, 168 insertions(+), 30 deletions(-) diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index 534f645..7f7e512 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -55,6 +55,89 @@ void writeAppendDebugFile(const std::string& label, } } +size_t countSubstring(const std::string& haystack, const std::string& needle) { + size_t count = 0; + size_t pos = 0; + while ((pos = haystack.find(needle, pos)) != std::string::npos) { + ++count; + pos += needle.size(); + } + return count; +} + +void logMultiTurnAppendPlan(const char* path_label, + const std::string& conversation_id, + size_t turn_number, + bool is_first_turn, + const std::string& full_prompt, + const std::string& text_to_append, + int full_tokens, + int append_tokens, + const char* input_mode) { + static constexpr const char* kUserTurnMarker = "<|im_start|>user"; + const size_t user_markers = countSubstring(full_prompt, kUserTurnMarker); + const size_t full_chars = full_prompt.size(); + const size_t append_chars = text_to_append.size(); + const int saved_tokens = full_tokens - append_tokens; + const int saved_pct = (full_tokens > 0) ? static_cast((saved_tokens * 100LL) / full_tokens) : 0; + + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " session=" << conversation_id + << " turn=" << turn_number + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " input=" << input_mode + << " user_markers=" << user_markers + << " full_chars=" << full_chars + << " full_tokens=" << full_tokens + << " append_chars=" << append_chars + << " append_tokens=" << append_tokens; + if (!is_first_turn) { + std::cout << " saved_tokens=" << saved_tokens << " saved_pct=" << saved_pct << "%"; + } + std::cout << std::endl; + + if (!is_first_turn) { + std::cout << "[InferenceEngine][MultiTurnDebug] delta_head: " + << text_to_append.substr(0, std::min(size_t(240), append_chars)) << std::endl; + } +} + +void logMultiTurnAfterAppend(const char* path_label, + size_t seq_before, + size_t seq_after, + bool is_done) { + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " seq_before=" << seq_before + << " seq_after=" << seq_after + << " seq_delta=" << (seq_after > seq_before ? seq_after - seq_before : 0) + << " is_done=" << (is_done ? "true" : "false") << std::endl; + if (is_done && seq_after <= seq_before) { + std::cerr << "[InferenceEngine][MultiTurnDebug] WARNING: generator IsDone after append " + << "with no new sequence tokens — generation loop will produce 0 output" + << std::endl; + } +} + +void logMultiTurnCompleted(const char* path_label, + size_t turn_number, + size_t seq_before, + size_t seq_after, + int generated_tokens, + int streamed_tokens) { + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " turn=" << turn_number + << " completed generated_tokens=" << generated_tokens + << " seq_before=" << seq_before + << " seq_after=" << seq_after; + if (streamed_tokens >= 0) { + std::cout << " streamed_tokens=" << streamed_tokens; + } + if (generated_tokens == 0) { + std::cout << " WARNING=zero_output"; + } + std::cout << std::endl; +} + } // namespace InferenceEngine::InferenceEngine(const std::string& model_path, int ctx_size) @@ -869,18 +952,11 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i } const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); - std::cout << "[InferenceEngine] Multi-turn turn=" << (session.turn_count + 1) - << " session=" << conversation_id - << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") - << " append_chars=" << text_to_append.length() - << " append_tokens=" << append_token_count - << " full_tokens=" << countTokens(full_prompt) << std::endl; - if (!is_first_turn) { - std::cout << "[InferenceEngine] Multi-turn delta (first 200): " - << text_to_append.substr(0, std::min(size_t(200), text_to_append.length())) - << std::endl; - } + logMultiTurnAppendPlan("text", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + "AppendTokens"); writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", conversation_id, @@ -889,6 +965,8 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i const size_t seq_before = session.generator->GetSequenceCount(0); appendPromptText(*session.generator, text_to_append); + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("text", seq_before, seq_after_append, session.generator->IsDone()); int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; if (model_context_length_ > 0 && total_max_length > model_context_length_) { @@ -923,6 +1001,9 @@ std::string InferenceEngine::completeMultiTurn(const std::string& conversation_i session.turn_count++; + logMultiTurnCompleted("text", session.turn_count, seq_before, output_count, + generated_token_count, -1); + if (out_timing != nullptr) { auto total_duration = std::chrono::duration_cast(end_time - start_time); auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); @@ -970,17 +1051,22 @@ void InferenceEngine::streamMultiTurn(const std::string& conversation_id, throw std::runtime_error("Multi-turn request produced an empty delta prompt"); } - std::cout << "[InferenceEngine] Multi-turn stream turn=" << (session.turn_count + 1) - << " session=" << conversation_id - << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") - << " append_tokens=" << countTokens(text_to_append) << std::endl; + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + + logMultiTurnAppendPlan("text-stream", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + "AppendTokens"); writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", conversation_id, session.turn_count + 1, text_to_append); + const size_t seq_before = session.generator->GetSequenceCount(0); appendPromptText(*session.generator, text_to_append); + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("text-stream", seq_before, seq_after_append, session.generator->IsDone()); int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; if (model_context_length_ > 0 && total_max_length > model_context_length_) { @@ -1027,6 +1113,13 @@ void InferenceEngine::streamMultiTurn(const std::string& conversation_id, accumulated_output = applyStopSequences(accumulated_output, params); session.turn_count++; + const size_t seq_after = session.generator->GetSequenceCount(0); + const int generated_token_count = (seq_after > seq_before) + ? static_cast(seq_after - seq_before) + : 0; + logMultiTurnCompleted("text-stream", session.turn_count, seq_before, seq_after, + generated_token_count, -1); + std::cout << "[InferenceEngine] Multi-turn stream completed turn=" << session.turn_count << " token_count=" << session.generator->TokenCount() << std::endl; } catch (const std::exception& e) { @@ -1150,14 +1243,14 @@ std::string InferenceEngine::completeMultiTurnMultimodal(const std::string& conv const std::vector& images_for_process = is_first_turn ? all_images : new_turn_images; - std::cout << "[InferenceEngine] Multimodal multi-turn turn=" << (active_session.turn_count + 1) - << " session=" << conversation_id - << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") - << " append_chars=" << text_to_append.length() - << " append_tokens=" << countTokens(text_to_append) - << " new_turn_images=" << new_turn_images.size() - << " process_images=" << images_for_process.size() - << " full_tokens=" << countTokens(full_prompt) << std::endl; + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + const char* input_mode = + (is_first_turn && !images_for_process.empty()) ? "SetInputs" : "AppendTokens"; + + logMultiTurnAppendPlan("vlm", conversation_id, active_session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + input_mode); writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", conversation_id, @@ -1174,6 +1267,9 @@ std::string InferenceEngine::completeMultiTurnMultimodal(const std::string& conv appendPromptText(*active_session.generator, text_to_append); } + const size_t seq_after_append = active_session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("vlm", seq_before, seq_after_append, active_session.generator->IsDone()); + int total_max_length = static_cast(active_session.generator->GetSequenceCount(0)) + params.max_length; if (model_context_length_ > 0 && total_max_length > model_context_length_) { total_max_length = model_context_length_; @@ -1207,6 +1303,9 @@ std::string InferenceEngine::completeMultiTurnMultimodal(const std::string& conv active_session.turn_count++; + logMultiTurnCompleted("vlm", active_session.turn_count, seq_before, output_count, + generated_token_count, -1); + if (out_timing != nullptr) { auto total_duration = std::chrono::duration_cast(end_time - start_time); auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); @@ -1273,18 +1372,25 @@ void InferenceEngine::streamMultiTurnMultimodal(const std::string& conversation_ const std::vector& images_for_process = is_first_turn ? all_images : new_turn_images; - std::cout << "[InferenceEngine] Multimodal multi-turn stream turn=" << (session.turn_count + 1) - << " session=" << conversation_id - << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") - << " append_tokens=" << countTokens(text_to_append) - << " new_turn_images=" << new_turn_images.size() - << " process_images=" << images_for_process.size() << std::endl; + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + const char* input_mode = + (is_first_turn && !images_for_process.empty()) ? "SetInputs" : "AppendTokens"; + + logMultiTurnAppendPlan("vlm-stream", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + input_mode); + std::cout << "[InferenceEngine][MultiTurnDebug] vlm-stream new_turn_images=" + << new_turn_images.size() << " process_images=" << images_for_process.size() + << std::endl; writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", conversation_id, session.turn_count + 1, text_to_append); + const size_t seq_before = session.generator->GetSequenceCount(0); + if (is_first_turn && !images_for_process.empty()) { auto oga_images = loadOgaImages(images_for_process); auto inputs = processor_->ProcessImages(text_to_append.c_str(), oga_images.get()); @@ -1293,6 +1399,9 @@ void InferenceEngine::streamMultiTurnMultimodal(const std::string& conversation_ appendPromptText(*session.generator, text_to_append); } + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("vlm-stream", seq_before, seq_after_append, session.generator->IsDone()); + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; if (model_context_length_ > 0 && total_max_length > model_context_length_) { total_max_length = model_context_length_; @@ -1338,6 +1447,13 @@ void InferenceEngine::streamMultiTurnMultimodal(const std::string& conversation_ accumulated_output = applyStopSequences(accumulated_output, params); session.turn_count++; + const size_t seq_after = session.generator->GetSequenceCount(0); + const int generated_token_count = (seq_after > seq_before) + ? static_cast(seq_after - seq_before) + : 0; + logMultiTurnCompleted("vlm-stream", session.turn_count, seq_before, seq_after, + generated_token_count, -1); + std::cout << "[InferenceEngine] Multimodal multi-turn stream completed turn=" << session.turn_count << std::endl; } catch (const std::exception& e) { diff --git a/src/server.cpp b/src/server.cpp index f05337b..b5fa202 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -134,6 +134,27 @@ std::vector extract_images_from_last_user_message(const json& reque return images; } +void logOpenAIChatRequestSummary(const ChatCompletionRequest& chat_req, bool multimodal) { + std::ostringstream role_seq; + for (size_t i = 0; i < chat_req.messages.size(); ++i) { + if (i > 0) { + role_seq << " -> "; + } + role_seq << chat_req.messages[i].role; + if (chat_req.messages[i].content.find("") != std::string::npos) { + role_seq << "(tool_response)"; + } + } + + const size_t tool_count = chat_req.tools.is_array() ? chat_req.tools.size() : 0; + std::cout << "[Server][ChatDebug] client=openai-compat multimodal=" << (multimodal ? 1 : 0) + << " messages=" << chat_req.messages.size() + << " tools=" << tool_count + << " stream=" << chat_req.stream + << " session=" << kDefaultConversationId + << " roles=" << role_seq.str() << std::endl; +} + } // namespace RyzenAIServer::RyzenAIServer(const CommandLineArgs& args) @@ -688,6 +709,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: return; } + logOpenAIChatRequestSummary(chat_req, inference_engine_->isMultimodal()); + // VLM models always use the multimodal multi-turn path (KV cache reuse on // text-only follow-ups). Only images in the latest user turn are processed. if (inference_engine_->isMultimodal()) { @@ -719,7 +742,6 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::cout << "[Server DEBUG] Prompt (first 500 chars): " << prompt.substr(0, std::min(size_t(500), prompt.length())) << std::endl; // Text chat always uses the single default multi-turn AppendTokens session. - // (Tool definitions in the request do not disable this; only post-hoc tool parsing.) const bool use_multi_turn = true; const std::string session_id = kDefaultConversationId; From c57243cc045c1039f2d323ae2de627a24416830c Mon Sep 17 00:00:00 2001 From: shili9 Date: Wed, 12 Aug 2026 21:59:59 -0500 Subject: [PATCH 6/6] update json format improvement --- src/server.cpp | 195 +++++++++---------------------------------------- 1 file changed, 35 insertions(+), 160 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index b5fa202..99901e3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -12,6 +12,33 @@ namespace ryzenai { namespace { +std::string escapeJsonString(const std::string& s) { + static constexpr char hex[] = "0123456789abcdef"; + std::string escaped; + escaped.reserve(s.size() + 8); + for (unsigned char c : s) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\b': escaped += "\\b"; break; + case '\f': escaped += "\\f"; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: + if (c < 0x20) { + escaped += "\\u00"; + escaped += hex[(c >> 4) & 0x0f]; + escaped += hex[c & 0x0f]; + } else { + escaped.push_back(static_cast(c)); + } + break; + } + } + return escaped; +} + // Minimal base64 decoder for image data URLs. std::string base64_decode(const std::string& input) { static const std::string chars = @@ -425,30 +452,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -529,30 +534,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // After generation completes, do a final flush to catch any remaining buffered content auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -798,30 +781,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -908,30 +869,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // This handles the case where the last few tokens didn't trigger processing due to buffer size auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings (reused from above) auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -972,17 +911,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: for (const auto& tool_call : extracted_tool_calls) { // Escape arguments for JSON std::string tool_call_args = tool_call.arguments.dump(); - std::string escaped_args = tool_call_args; - size_t pos = 0; - while ((pos = escaped_args.find('\\', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_args.find('"', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\""); - pos += 2; - } + std::string escaped_args = escapeJsonString(tool_call_args); std::string tool_call_chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + @@ -1163,19 +1092,7 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, chat_req.top_k, chat_req.repeat_penalty, chat_req.stop); auto escapeJson = [](const std::string& str) -> std::string { - std::string e; - e.reserve(str.size() + 8); - for (char c : str) { - switch (c) { - case '\\': e += "\\\\"; break; - case '"': e += "\\\""; break; - case '\n': e += "\\n"; break; - case '\r': e += "\\r"; break; - case '\t': e += "\\t"; break; - default: e += c; break; - } - } - return e; + return escapeJsonString(str); }; const std::string model_id = model_id_; @@ -1357,28 +1274,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon // Generate and send tokens in real-time inference_engine_->streamComplete(prompt, params, [&sink, &full_response](const std::string& token, bool is_final) -> bool { - // Escape special characters for JSON - std::string escaped_token = token; - size_t pos = 0; - while ((pos = escaped_token.find('\\', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('"', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\n', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\r', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_token = escapeJsonString(token); // Accumulate unescaped token for final response full_response += token; @@ -1399,28 +1295,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon std::cout << "[Server] Token generation completed, sending final events" << std::endl; - // Escape full_response for JSON - std::string escaped_full_response = full_response; - size_t pos = 0; - while ((pos = escaped_full_response.find('\\', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('"', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\n', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\r', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_full_response = escapeJsonString(full_response); // Send response.completed event std::string completed_time = std::to_string(std::time(nullptr));