From 87cbacc95ed2002de925361287c5e9e9f43cfc81 Mon Sep 17 00:00:00 2001 From: Paresh Bhagat Date: Sun, 16 Aug 2026 21:03:34 +0530 Subject: [PATCH 1/7] example: edge-ai: Restructure edge-ai into focused pipeline files - Extract pipeline files: audio_enhancement, stft_istft, tvm, audio_utils, pipeline_common - Rename GenericTaskClient -> DspTaskClient; remove interactive CLI mode - Add pipeline_type field for dispatch; remove pipeline_id and model_config - Move model params (hop_size, model_elems, total_frames, batch_n) to stage parameters - Add dsp_config (proc_id, endpoint) to JSON; remove all hardcoded RPMsg constants - Size STFT/ISTFT DMA buffers independently from their respective stage parameters Signed-off-by: Paresh Bhagat --- example/edge-ai/CMakeLists.txt | 11 +- .../include/audio_enhancement_pipeline.h | 14 + example/edge-ai/include/audio_utils.h | 11 + ...eneric_task_client.h => dsp_task_client.h} | 22 +- example/edge-ai/include/pipeline_common.h | 76 ++ example/edge-ai/include/pipeline_manager.h | 71 +- example/edge-ai/include/stft_istft_pipeline.h | 12 + example/edge-ai/include/tvm_pipeline.h | 11 + .../pipeline_audio_enhancement.json | 42 +- .../json_files/pipeline_stft_istft.json | 40 +- .../json_files/pipeline_tvm_inference.json | 2 +- .../src/audio_enhancement_pipeline.cpp | 392 ++++++ example/edge-ai/src/audio_utils.cpp | 101 ++ ...ic_task_client.cpp => dsp_task_client.cpp} | 50 +- example/edge-ai/src/main.cpp | 17 +- example/edge-ai/src/pipeline_common.cpp | 116 ++ example/edge-ai/src/pipeline_manager.cpp | 1197 +---------------- example/edge-ai/src/stft_istft_pipeline.cpp | 345 +++++ example/edge-ai/src/tvm_pipeline.cpp | 102 ++ 19 files changed, 1385 insertions(+), 1247 deletions(-) create mode 100644 example/edge-ai/include/audio_enhancement_pipeline.h create mode 100644 example/edge-ai/include/audio_utils.h rename example/edge-ai/include/{generic_task_client.h => dsp_task_client.h} (84%) create mode 100644 example/edge-ai/include/pipeline_common.h create mode 100644 example/edge-ai/include/stft_istft_pipeline.h create mode 100644 example/edge-ai/include/tvm_pipeline.h create mode 100644 example/edge-ai/src/audio_enhancement_pipeline.cpp create mode 100644 example/edge-ai/src/audio_utils.cpp rename example/edge-ai/src/{generic_task_client.cpp => dsp_task_client.cpp} (89%) create mode 100644 example/edge-ai/src/pipeline_common.cpp create mode 100644 example/edge-ai/src/stft_istft_pipeline.cpp create mode 100644 example/edge-ai/src/tvm_pipeline.cpp diff --git a/example/edge-ai/CMakeLists.txt b/example/edge-ai/CMakeLists.txt index 4ebba6f..36177fd 100644 --- a/example/edge-ai/CMakeLists.txt +++ b/example/edge-ai/CMakeLists.txt @@ -27,8 +27,13 @@ message(STATUS "Using TVM_ROOT: ${TVM_ROOT}") set(EDGE_AI_SOURCES src/main.cpp src/tvm_inference_client.cpp - src/generic_task_client.cpp + src/dsp_task_client.cpp src/pipeline_manager.cpp + src/pipeline_common.cpp + src/audio_utils.cpp + src/tvm_pipeline.cpp + src/stft_istft_pipeline.cpp + src/audio_enhancement_pipeline.cpp ) # Create the executable @@ -69,9 +74,6 @@ pkg_check_modules(JSON_C REQUIRED json-c) # Add json-c include directory target_include_directories(rpmsg_inference_example PRIVATE ${JSON_C_INCLUDE_DIRS}) -# Find readline library for interactive mode with command line editing -find_library(READLINE_LIB readline REQUIRED) - # Find audio libraries for WAV file processing and playback find_library(SNDFILE_LIB sndfile REQUIRED) find_library(ALSA_LIB asound REQUIRED) @@ -81,7 +83,6 @@ target_link_libraries(rpmsg_inference_example ${TVM_RUNTIME_LIB} ti_rpmsg_dma ${JSON_C_LIBRARIES} - ${READLINE_LIB} ${SNDFILE_LIB} ${ALSA_LIB} pthread diff --git a/example/edge-ai/include/audio_enhancement_pipeline.h b/example/edge-ai/include/audio_enhancement_pipeline.h new file mode 100644 index 0000000..c00d8c6 --- /dev/null +++ b/example/edge-ai/include/audio_enhancement_pipeline.h @@ -0,0 +1,14 @@ +#ifndef AUDIO_ENHANCEMENT_PIPELINE_H +#define AUDIO_ENHANCEMENT_PIPELINE_H + +#include "pipeline_manager.h" +#include "dsp_task_client.h" +#include "tvm_inference_client.h" + +PipelineManager::CommandResult run_audio_enhancement_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug); + +#endif // AUDIO_ENHANCEMENT_PIPELINE_H diff --git a/example/edge-ai/include/audio_utils.h b/example/edge-ai/include/audio_utils.h new file mode 100644 index 0000000..43b348f --- /dev/null +++ b/example/edge-ai/include/audio_utils.h @@ -0,0 +1,11 @@ +#ifndef AUDIO_UTILS_H +#define AUDIO_UTILS_H + +#include +#include +#include + +bool loadAudioFile(const std::string& filename, std::vector& audio_data); +bool saveAudioFile(const std::string& filename, const std::vector& audio_data); + +#endif // AUDIO_UTILS_H diff --git a/example/edge-ai/include/generic_task_client.h b/example/edge-ai/include/dsp_task_client.h similarity index 84% rename from example/edge-ai/include/generic_task_client.h rename to example/edge-ai/include/dsp_task_client.h index a3833d6..8f976f7 100644 --- a/example/edge-ai/include/generic_task_client.h +++ b/example/edge-ai/include/dsp_task_client.h @@ -1,5 +1,5 @@ -#ifndef GENERIC_TASK_CLIENT_H -#define GENERIC_TASK_CLIENT_H +#ifndef DSP_TASK_CLIENT_H +#define DSP_TASK_CLIENT_H #include #include @@ -17,7 +17,7 @@ extern "C" { * Message type determines which struct and processing logic to use. * Uses zero-copy approach with TVM shared memory regions. */ -class GenericTaskClient { +class DspTaskClient { public: struct ProcessingResult { bool success; @@ -26,8 +26,8 @@ class GenericTaskClient { std::string error_message; }; - GenericTaskClient(); - ~GenericTaskClient(); + DspTaskClient(); + ~DspTaskClient(); /** * @brief Initialize the Generic Task client @@ -35,7 +35,9 @@ class GenericTaskClient { * @param max_output_size Maximum output buffer size in bytes * @return true on success, false on failure */ - bool initialize(uint32_t max_input_size = 1024*1024, uint32_t max_output_size = 1024*1024); + bool initialize(int proc_id, int endpoint, + uint32_t max_input_size = 1024*1024, + uint32_t max_output_size = 1024*1024); /** * @brief Generic processing function @@ -83,13 +85,11 @@ class GenericTaskClient { private: // RPMsg communication int rpmsg_fd_; + int proc_id_; + int endpoint_; bool initialized_; uint32_t sequence_number_; - // Shared memory addresses (zero-copy approach) - uint32_t shared_input_addr_; // TVM staging physical address (0xa3000000) - uint32_t shared_output_addr_; // TVM result physical address (0xabc00000) - // Internal methods bool open_rpmsg_device(); void close_rpmsg_device(); @@ -98,4 +98,4 @@ class GenericTaskClient { std::string get_error_string(int32_t error_code); }; -#endif // GENERIC_TASK_CLIENT_H +#endif // DSP_TASK_CLIENT_H diff --git a/example/edge-ai/include/pipeline_common.h b/example/edge-ai/include/pipeline_common.h new file mode 100644 index 0000000..9d9bb49 --- /dev/null +++ b/example/edge-ai/include/pipeline_common.h @@ -0,0 +1,76 @@ +#ifndef PIPELINE_COMMON_H +#define PIPELINE_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "dmabuf.h" +#include "fw_loader.h" +} + +class PipelineError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +std::string hex_address(uint64_t address); + +class DmaBuffer { +public: + DmaBuffer(size_t bytes, std::string_view purpose); + ~DmaBuffer(); + DmaBuffer(const DmaBuffer&) = delete; + DmaBuffer& operator=(const DmaBuffer&) = delete; + + dma_buf_params* operator->() noexcept { return ¶ms_; } + const dma_buf_params* operator->() const noexcept { return ¶ms_; } + + template + T* data() noexcept { return reinterpret_cast(params_.kern_addr); } + + void begin_cpu_access() const; + void end_cpu_access() const; + +private: + void sync(int operation) const; + + dma_buf_params params_{}; + bool allocated_{false}; +}; + +class AudioStream { +public: + AudioStream() noexcept; + ~AudioStream(); + AudioStream(const AudioStream&) = delete; + AudioStream& operator=(const AudioStream&) = delete; + + void send_frame(uint8_t direction, const void* pcm, size_t bytes) noexcept; + +private: + static std::array make_header(uint8_t direction, + uint32_t pcm_bytes) noexcept; + static void write_u32_le(std::array& destination, + size_t offset, uint32_t value) noexcept; + void open() noexcept; + bool send_all(const void* data, size_t bytes) noexcept; + static void close_fd(int& descriptor) noexcept; + + static constexpr std::string_view socket_path_{"/tmp/edge-ai-speech.sock"}; + static_assert(socket_path_.size() < sizeof(sockaddr_un{}.sun_path), + "Audio stream socket path is too long"); + int server_{-1}; + int client_{-1}; +}; + +#endif // PIPELINE_COMMON_H diff --git a/example/edge-ai/include/pipeline_manager.h b/example/edge-ai/include/pipeline_manager.h index 9fe330e..80a70c1 100644 --- a/example/edge-ai/include/pipeline_manager.h +++ b/example/edge-ai/include/pipeline_manager.h @@ -4,10 +4,9 @@ #include #include #include -#include #include #include "tvm_inference_client.h" -#include "generic_task_client.h" +#include "dsp_task_client.h" extern "C" { #include @@ -22,21 +21,25 @@ class PipelineManager { QUIT }; - // JSON Pipeline Stage struct PipelineStage { std::string stage_id; - std::string service; // "generic" or "tvm" - std::string message_type; // "C7X_MSG_*" or "TVM_INFERENCE" - std::map parameters; // Stage-specific parameters + std::string service; + std::string message_type; + std::map parameters; }; + struct DspConfig { + int proc_id = 0; + int endpoint = 0; + }; struct PipelineConfig { - std::string pipeline_id; + std::string pipeline_type; std::string description; std::string input_file; std::string artifacts_path; std::vector stages; + DspConfig dsp_config; bool loaded; PipelineConfig() : loaded(false) {} @@ -48,12 +51,6 @@ class PipelineManager { TENSOR_BIN }; - enum class PipelineMode { - STFT_ISTFT, // --ISTFT : STFT -> ISTFT only - TVM_ONLY, // --TVM : TVM inference only (BIN input) - FULL // --full : STFT -> TVM -> ISTFT - }; - struct State { bool artifacts_loaded; PipelineConfig pipeline_config; @@ -64,70 +61,24 @@ class PipelineManager { InputType input_type; bool input_configured; - void* tvm_staging_buffer; - void* tvm_result_buffer; - size_t staging_buffer_size; - size_t result_buffer_size; - State() : input_type(InputType::UNKNOWN) {} }; PipelineManager(); ~PipelineManager(); bool initialize(); - int run(); - int run_direct(PipelineMode mode, const std::string& input_file, const std::string& artifacts_path = ""); int run_from_json_file(const std::string& json_file_path); void set_debug(bool enable) { debug_ = enable; } private: std::shared_ptr tvm_client_; - std::unique_ptr generic_client_; + std::unique_ptr generic_client_; State state_; - std::vector deint_output_data_; - std::vector inter_input_data_; bool initialized_; bool debug_ = false; - std::string app_name_; - - struct CommandInfo { - std::string description; - std::vector examples; - std::function&)> handler; - }; - - std::map commands_; - - void initializeCommands(); - void printWelcome(); - void printPrompt(); - std::vector parseCommand(const std::string& input); - CommandResult executeCommand(const std::vector& tokens); - - CommandResult handleHelp(const std::vector& args); - CommandResult handlePipeline(const std::vector& args); - CommandResult handleTvmArtifacts(const std::vector& args); - CommandResult handleInput(const std::vector& args); - CommandResult handleShowPipeline(const std::vector& args); - CommandResult handleRun(const std::vector& args); - CommandResult handleStatus(const std::vector& args); - CommandResult handleQuit(const std::vector& args); bool validateConfiguration(); bool loadPipelineFromJson(const std::string& json_content); - - CommandResult executeTensorPipeline(); - CommandResult executeSequentialPipeline(); - - bool loadAudioFile(const std::string& filename, std::vector& audio_data); - bool loadBinTensor(const std::string& filename, std::vector& tensor_data); - bool saveTensorFile(const std::string& filename, const std::vector& tensor_data); - bool playAudioData(const std::vector& audio_data); - bool saveAudioFile(const std::string& filename, const std::vector& audio_data); - std::string getCurrentPrompt(); - void registerCommand(const std::string& name, const std::string& description, - const std::vector& examples, - std::function&)> handler); }; #endif // PIPELINE_MANAGER_H diff --git a/example/edge-ai/include/stft_istft_pipeline.h b/example/edge-ai/include/stft_istft_pipeline.h new file mode 100644 index 0000000..9d1c1d4 --- /dev/null +++ b/example/edge-ai/include/stft_istft_pipeline.h @@ -0,0 +1,12 @@ +#ifndef STFT_ISTFT_PIPELINE_H +#define STFT_ISTFT_PIPELINE_H + +#include "pipeline_manager.h" +#include "dsp_task_client.h" + +PipelineManager::CommandResult run_stft_istft_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + bool debug); + +#endif // STFT_ISTFT_PIPELINE_H diff --git a/example/edge-ai/include/tvm_pipeline.h b/example/edge-ai/include/tvm_pipeline.h new file mode 100644 index 0000000..90e1324 --- /dev/null +++ b/example/edge-ai/include/tvm_pipeline.h @@ -0,0 +1,11 @@ +#ifndef TVM_PIPELINE_H +#define TVM_PIPELINE_H + +#include "pipeline_manager.h" +#include "tvm_inference_client.h" + +PipelineManager::CommandResult run_tvm_pipeline( + PipelineManager::State& state, + TvmInferenceClient& tvm_client); + +#endif // TVM_PIPELINE_H diff --git a/example/edge-ai/json_files/pipeline_audio_enhancement.json b/example/edge-ai/json_files/pipeline_audio_enhancement.json index 20273fd..54c860b 100644 --- a/example/edge-ai/json_files/pipeline_audio_enhancement.json +++ b/example/edge-ai/json_files/pipeline_audio_enhancement.json @@ -1,26 +1,60 @@ { - "pipeline_id": "audio_gcrn_pipeline", + "pipeline_type": "audio_enhancement", "description": "GCRN 3-stage audio pipeline: STFT analyze -> TVM inference -> ISTFT synthesize", "input_file": "/usr/share/tvm_inference/input/input_audio.wav", "artifacts_path": "/usr/share/tvm_inference/artifacts/", + "dsp_config": { + "proc_id": 8, + "endpoint": 13 + }, "stages": [ { "stage_id": "stft_analysis", "service": "generic", "message_type": "C7X_MSG_STFT_ANALYZE", - "parameters": {} + "parameters": { + "hop_size": 160, + "model_elems": 322, + "total_frames": 401, + "batch_n": 64 + } + }, + { + "stage_id": "deinterleave", + "service": "generic", + "message_type": "C7X_DEINTERLEAVE_MSG_ANALYZE", + "parameters": { + "fft_size": 320, + "flag": 0 + } }, { "stage_id": "tvm_inference", "service": "tvm", "message_type": "TVM_INFERENCE", - "parameters": {} + "parameters": { + "input_shape": "1,2,401,161" + } + }, + { + "stage_id": "interleave", + "service": "generic", + "message_type": "C7X_DEINTERLEAVE_MSG_ANALYZE", + "parameters": { + "fft_size": 320, + "flag": 1 + } }, { "stage_id": "stft_synthesis", "service": "generic", "message_type": "C7X_MSG_ISTFT_SYNTHESIZE", - "parameters": {} + "parameters": { + "hop_size": 160, + "model_elems": 322, + "total_frames": 401, + "batch_n": 64 + } } ] } diff --git a/example/edge-ai/json_files/pipeline_stft_istft.json b/example/edge-ai/json_files/pipeline_stft_istft.json index 5a35c50..fb848bc 100644 --- a/example/edge-ai/json_files/pipeline_stft_istft.json +++ b/example/edge-ai/json_files/pipeline_stft_istft.json @@ -1,19 +1,51 @@ { - "pipeline_id": "audio_stft_istft", - "description": "STFT round-trip pipeline: STFT analyze -> ISTFT synthesize (no TVM)", + "pipeline_type": "stft_istft", + "description": "STFT round-trip pipeline: STFT analyze -> deinterleave -> interleave -> ISTFT synthesize", "input_file": "/usr/share/tvm_inference/input/input_audio.wav", + "dsp_config": { + "proc_id": 8, + "endpoint": 13 + }, "stages": [ { "stage_id": "stft_analysis", "service": "generic", "message_type": "C7X_MSG_STFT_ANALYZE", - "parameters": {} + "parameters": { + "hop_size": 160, + "model_elems": 322, + "total_frames": 401, + "batch_n": 64 + } + }, + { + "stage_id": "deinterleave", + "service": "generic", + "message_type": "C7X_DEINTERLEAVE_MSG_ANALYZE", + "parameters": { + "fft_size": 320, + "flag": 0 + } + }, + { + "stage_id": "interleave", + "service": "generic", + "message_type": "C7X_DEINTERLEAVE_MSG_ANALYZE", + "parameters": { + "fft_size": 320, + "flag": 1 + } }, { "stage_id": "stft_synthesis", "service": "generic", "message_type": "C7X_MSG_ISTFT_SYNTHESIZE", - "parameters": {} + "parameters": { + "hop_size": 160, + "model_elems": 322, + "total_frames": 401, + "batch_n": 64 + } } ] } diff --git a/example/edge-ai/json_files/pipeline_tvm_inference.json b/example/edge-ai/json_files/pipeline_tvm_inference.json index 77ebc87..67a29ee 100644 --- a/example/edge-ai/json_files/pipeline_tvm_inference.json +++ b/example/edge-ai/json_files/pipeline_tvm_inference.json @@ -1,5 +1,5 @@ { - "pipeline_id": "tensor_tvm_only", + "pipeline_type": "tvm_only", "description": "Direct TVM inference pipeline: BIN tensor input -> TVM inference -> Binary output", "input_file": "/usr/share/tvm_inference/input/gcrn_fixed_input.bin", "artifacts_path": "/usr/share/tvm_inference/artifacts/", diff --git a/example/edge-ai/src/audio_enhancement_pipeline.cpp b/example/edge-ai/src/audio_enhancement_pipeline.cpp new file mode 100644 index 0000000..444c73e --- /dev/null +++ b/example/edge-ai/src/audio_enhancement_pipeline.cpp @@ -0,0 +1,392 @@ +#include "audio_enhancement_pipeline.h" +#include "pipeline_common.h" +#include "audio_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +size_t require_param(const std::map& params, + const char* key, const char* stage) +{ + auto it = params.find(key); + if (it == params.end()) + throw PipelineError{std::string{"STFT stage missing required parameter: "} + key + + " (stage: " + stage + ")"}; + int v = std::stoi(it->second); + if (v <= 0) + throw PipelineError{std::string{"Parameter must be positive: "} + key}; + return static_cast(v); +} + +} // namespace + +PipelineManager::CommandResult run_audio_enhancement_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + TvmInferenceClient& tvm_client, + bool debug) +{ + try { + if (state.input_type != PipelineManager::InputType::AUDIO_WAV) + throw PipelineError{"Unknown input type"}; + + // Find required stages + const PipelineManager::PipelineStage* stft_stage_ptr = nullptr; + const PipelineManager::PipelineStage* deint_stage_ptr = nullptr; + const PipelineManager::PipelineStage* tvm_stage_ptr = nullptr; + const PipelineManager::PipelineStage* inter_stage_ptr = nullptr; + const PipelineManager::PipelineStage* istft_stage_ptr = nullptr; + + for (const auto& stage : state.pipeline_config.stages) { + if (stage.message_type == "C7X_MSG_STFT_ANALYZE") stft_stage_ptr = &stage; + else if (stage.message_type == "C7X_MSG_ISTFT_SYNTHESIZE") istft_stage_ptr = &stage; + else if (stage.message_type == "TVM_INFERENCE") tvm_stage_ptr = &stage; + else if (stage.message_type == "C7X_DEINTERLEAVE_MSG_ANALYZE") { + auto it = stage.parameters.find("flag"); + if (it != stage.parameters.end() && it->second == "0") + deint_stage_ptr = &stage; + else + inter_stage_ptr = &stage; + } + } + + if (!stft_stage_ptr || !istft_stage_ptr || !deint_stage_ptr || !inter_stage_ptr) + throw PipelineError{"audio_enhancement pipeline requires STFT, deinterleave, interleave and ISTFT stages"}; + + // Read loop/buffer parameters from STFT stage (drives STFT-side buffers and loop) + const auto& sp = stft_stage_ptr->parameters; + const size_t HOP_SIZE = require_param(sp, "hop_size", stft_stage_ptr->stage_id.c_str()); + const size_t MODEL_ELEMS = require_param(sp, "model_elems", stft_stage_ptr->stage_id.c_str()); + const size_t TOTAL_FRAMES = require_param(sp, "total_frames", stft_stage_ptr->stage_id.c_str()); + const size_t BATCH_N = require_param(sp, "batch_n", stft_stage_ptr->stage_id.c_str()); + const size_t PAD_FRAMES = TOTAL_FRAMES % BATCH_N; + const size_t NUM_BATCHES = (TOTAL_FRAMES + BATCH_N - 1) / BATCH_N; + + // Read parameters from ISTFT stage (drives ISTFT-side buffers — TVM output may differ) + const auto& ip = istft_stage_ptr->parameters; + const size_t ISTFT_HOP_SIZE = require_param(ip, "hop_size", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_MODEL_ELEMS = require_param(ip, "model_elems", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_TOTAL_FRAMES = require_param(ip, "total_frames", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_BATCH_N = require_param(ip, "batch_n", istft_stage_ptr->stage_id.c_str()); + + // STFT-side buffer sizes (buf1, buf2, buf5) + const size_t audio_batch_bytes = BATCH_N * HOP_SIZE * sizeof(int16_t); + const size_t spectral_stft_bytes = TOTAL_FRAMES * MODEL_ELEMS * sizeof(float); + + // ISTFT-side buffer sizes (buf3, buf4, buf6) — based on TVM output shape + const size_t audio_istft_batch_bytes = ISTFT_BATCH_N * ISTFT_HOP_SIZE * sizeof(int16_t); + const size_t spectral_istft_bytes = ISTFT_TOTAL_FRAMES * ISTFT_MODEL_ELEMS * sizeof(float); + + std::cout << "[App] STFT parameters: HOP=" << HOP_SIZE << " MODEL_ELEMS=" << MODEL_ELEMS + << " BATCH_N=" << BATCH_N << " TOTAL_FRAMES=" << TOTAL_FRAMES << std::endl; + std::cout << "[App] ISTFT parameters: HOP=" << ISTFT_HOP_SIZE << " MODEL_ELEMS=" << ISTFT_MODEL_ELEMS + << " BATCH_N=" << ISTFT_BATCH_N << " TOTAL_FRAMES=" << ISTFT_TOTAL_FRAMES << std::endl; + std::cout << "[App] STFT spectral buffer: " << spectral_stft_bytes << " bytes" << std::endl; + std::cout << "[App] ISTFT spectral buffer: " << spectral_istft_bytes << " bytes" << std::endl; + + // Load audio + std::vector audio_data; + if (!loadAudioFile(state.current_input_file, audio_data)) + throw PipelineError{"Failed to load input audio"}; + const size_t original_sample_count = audio_data.size(); + + if (!dsp_client.initialize(state.pipeline_config.dsp_config.proc_id, + state.pipeline_config.dsp_config.endpoint)) + throw PipelineError{"Failed to initialize DSP Task client"}; + + // Allocate DMA buffers + // STFT-side: buf1 (audio in), buf2 (STFT out), buf5 (deinterleave out) + // ISTFT-side: buf3 (interleave out), buf4 (ISTFT audio out), buf6 (interleave in) + DmaBuffer dma_buf1{audio_batch_bytes, "STFT input"}; + DmaBuffer dma_buf2{spectral_stft_bytes, "STFT output"}; + DmaBuffer dma_buf3{spectral_istft_bytes, "interleave output"}; + DmaBuffer dma_buf4{audio_istft_batch_bytes, "ISTFT output"}; + DmaBuffer dma_buf5{spectral_stft_bytes, "deinterleave output"}; + DmaBuffer dma_buf6{spectral_istft_bytes, "interleave input"}; + + std::cout << "[App] DMA buffers:" << std::endl; + std::cout << "[App] buf1 (STFT audio in): phys=0x" << std::hex << dma_buf1->phys_addr + << std::dec << " size=" << dma_buf1->size << std::endl; + std::cout << "[App] buf2 (STFT out/deint in): phys=0x" << std::hex << dma_buf2->phys_addr + << std::dec << " size=" << dma_buf2->size << std::endl; + std::cout << "[App] buf3 (int out/ISTFT in): phys=0x" << std::hex << dma_buf3->phys_addr + << std::dec << " size=" << dma_buf3->size << std::endl; + std::cout << "[App] buf4 (ISTFT audio out): phys=0x" << std::hex << dma_buf4->phys_addr + << std::dec << " size=" << dma_buf4->size << std::endl; + std::cout << "[App] buf5 (deint out): phys=0x" << std::hex << dma_buf5->phys_addr + << std::dec << " size=" << dma_buf5->size << std::endl; + std::cout << "[App] buf6 (interleave in): phys=0x" << std::hex << dma_buf6->phys_addr + << std::dec << " size=" << dma_buf6->size << std::endl; + + // Chunk calculation + const size_t total_frames = + (audio_data.size() + HOP_SIZE - 1) / HOP_SIZE; + audio_data.resize(total_frames * HOP_SIZE, int16_t{0}); + const size_t num_full_chunks = total_frames / TOTAL_FRAMES; + const size_t partial_frames = total_frames % TOTAL_FRAMES; + const size_t num_chunks = num_full_chunks + (partial_frames > 0 ? 1 : 0); + + std::cout << "[App] Full file processing:" << std::endl; + std::cout << "[App] Total frames: " << total_frames + << " | Full chunks: " << num_full_chunks + << " | Partial chunk: " << partial_frames << " real frames" + << " (zero-padded to " << TOTAL_FRAMES << ")" << std::endl; + + // Initialize TVM — read input shape from TVM stage parameters + if (tvm_stage_ptr && state.tvm_artifacts_configured && !tvm_client.is_initialized()) { + if (!tvm_client.initialize(state.tvm_artifacts_paths[0])) + throw PipelineError{"Failed to initialize TVM client"}; + + // Parse input_shape from TVM stage parameters e.g. "1,2,401,161" + std::vector input_shape; + auto shape_it = tvm_stage_ptr->parameters.find("input_shape"); + if (shape_it != tvm_stage_ptr->parameters.end()) { + std::istringstream ss(shape_it->second); + std::string token; + while (std::getline(ss, token, ',')) + input_shape.push_back(std::stoi(token)); + } else { + // Derive from model params as fallback + input_shape = {1, 2, + static_cast(TOTAL_FRAMES), + static_cast(MODEL_ELEMS / 2)}; + } + tvm_client.set_input_shape(input_shape); + std::cout << "[App] TVM initialized, input shape ["; + for (size_t i = 0; i < input_shape.size(); ++i) + std::cout << input_shape[i] << (i + 1 < input_shape.size() ? "," : ""); + std::cout << "]" << std::endl; + } + + std::vector processed_audio_data; + processed_audio_data.reserve(total_frames * HOP_SIZE); + + AudioStream audio_stream; + + uint64_t stft_out_base = dma_buf2->phys_addr; + uint64_t istft_src_base = dma_buf3->phys_addr; + + std::vector deint_output_data; + std::vector inter_input_data; + + auto t_total_start = std::chrono::steady_clock::now(); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) { + const size_t chunk_frame_offset = chunk_idx * TOTAL_FRAMES; + const size_t num_batches_chunk = NUM_BATCHES; + const size_t real_frames_chunk = (chunk_idx == num_full_chunks && partial_frames > 0) + ? partial_frames : TOTAL_FRAMES; + + auto t_chunk_start = std::chrono::steady_clock::now(); + + // Phase 1: STFT + auto t_stft_start = std::chrono::steady_clock::now(); + for (size_t batch_idx = 0; batch_idx < num_batches_chunk; batch_idx++) { + const size_t frames_this_batch = (batch_idx < NUM_BATCHES - 1) ? BATCH_N : PAD_FRAMES; + const size_t samples_this_batch = frames_this_batch * HOP_SIZE; + const size_t audio_offset = (chunk_frame_offset + batch_idx * BATCH_N) * HOP_SIZE; + const size_t audio_bytes = samples_this_batch * sizeof(int16_t); + const uint64_t spectral_offset = batch_idx * BATCH_N * MODEL_ELEMS * sizeof(float); + + dma_buf1.begin_cpu_access(); + std::fill_n(dma_buf1.data(), audio_batch_bytes, std::byte{}); + if (audio_offset < audio_data.size()) { + const size_t available = std::min(samples_this_batch, + audio_data.size() - audio_offset); + std::copy_n(audio_data.begin() + static_cast(audio_offset), + available, dma_buf1.data()); + } + dma_buf1.end_cpu_access(); + audio_stream.send_frame(0, dma_buf1.data(), audio_bytes); + + auto params = stft_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf1->phys_addr); + params["output_buffer"] = hex_address(stft_out_base + spectral_offset); + params["input_frame"] = std::to_string(frames_this_batch); + params["output_frame"] = std::to_string(frames_this_batch); + + if (debug) + std::cout << "[App] STFT batch " << (batch_idx+1) << "/" << num_batches_chunk + << ": " << frames_this_batch << " frames" + << " in=0x" << std::hex << dma_buf1->phys_addr + << " out=0x" << (stft_out_base + spectral_offset) << std::dec << std::endl; + + auto r = dsp_client.process("C7X_MSG_STFT_ANALYZE", params); + if (!r.success) + throw PipelineError{"STFT batch " + std::to_string(batch_idx + 1) + + " failed: " + r.error_message}; + } + double t_stft_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_stft_start).count() / 1000.0; + + // Phase 2: Deinterleave + if (debug) + std::cout << "[App] Deinterleave: 0x" << std::hex << dma_buf2->phys_addr + << " -> 0x" << dma_buf5->phys_addr << std::dec << std::endl; + { + auto params = deint_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf2->phys_addr); + params["output_buffer"] = hex_address(dma_buf5->phys_addr); + params["input_frame"] = std::to_string(TOTAL_FRAMES); + auto r = dsp_client.process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); + if (!r.success) + throw PipelineError{"Deinterleave failed: " + r.error_message}; + } + + // Phase 3: TVM + double t_tvm_ms = 0.0; + auto t_tvm_start = std::chrono::steady_clock::now(); + + deint_output_data.resize(spectral_stft_bytes / sizeof(float)); + inter_input_data.resize(spectral_istft_bytes / sizeof(float)); + + dma_buf5.begin_cpu_access(); + std::copy_n(dma_buf5.data(), deint_output_data.size(), + deint_output_data.begin()); + dma_buf5.end_cpu_access(); + + if (tvm_client.is_initialized()) { + if (!tvm_client.run_inference(deint_output_data, inter_input_data)) + throw PipelineError{"TVM inference failed"}; + t_tvm_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_tvm_start).count() / 1000.0; + } else { + inter_input_data = deint_output_data; + } + + if (inter_input_data.size() != spectral_istft_bytes / sizeof(float)) + throw PipelineError{"TVM output size does not match ISTFT spectral buffer"}; + + // Phase 4: Interleave + dma_buf6.begin_cpu_access(); + std::copy(inter_input_data.begin(), inter_input_data.end(), + dma_buf6.data()); + dma_buf6.end_cpu_access(); + + if (debug) + std::cout << "[App] Interleave: 0x" << std::hex << dma_buf6->phys_addr + << " -> 0x" << dma_buf3->phys_addr << std::dec << std::endl; + { + auto params = inter_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf6->phys_addr); + params["output_buffer"] = hex_address(dma_buf3->phys_addr); + params["input_frame"] = std::to_string(TOTAL_FRAMES); + auto r = dsp_client.process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); + if (!r.success) + throw PipelineError{"Interleave failed: " + r.error_message}; + } + + // Phase 5: ISTFT + const size_t ISTFT_PAD_FRAMES = ISTFT_TOTAL_FRAMES % ISTFT_BATCH_N; + const size_t ISTFT_NUM_BATCHES = (ISTFT_TOTAL_FRAMES + ISTFT_BATCH_N - 1) / ISTFT_BATCH_N; + auto t_istft_start = std::chrono::steady_clock::now(); + size_t real_samples_remaining = real_frames_chunk * ISTFT_HOP_SIZE; + for (size_t batch_idx = 0; batch_idx < ISTFT_NUM_BATCHES; batch_idx++) { + const size_t frames_this_batch = (batch_idx < ISTFT_NUM_BATCHES - 1) ? ISTFT_BATCH_N : ISTFT_PAD_FRAMES; + const size_t samples_this_batch = frames_this_batch * ISTFT_HOP_SIZE; + const size_t audio_offset = (chunk_frame_offset + batch_idx * ISTFT_BATCH_N) * ISTFT_HOP_SIZE; + const uint64_t spectral_offset = batch_idx * ISTFT_BATCH_N * ISTFT_MODEL_ELEMS * sizeof(float); + + auto params = istft_stage_ptr->parameters; + params["input_buffer"] = hex_address(istft_src_base + spectral_offset); + params["output_buffer"] = hex_address(dma_buf4->phys_addr); + params["input_frame"] = std::to_string(frames_this_batch); + params["output_frame"] = std::to_string(frames_this_batch); + + if (debug) + std::cout << "[App] ISTFT batch " << (batch_idx+1) << "/" << num_batches_chunk + << ": " << frames_this_batch << " frames" + << " in=0x" << std::hex << (istft_src_base + spectral_offset) + << " out=0x" << dma_buf4->phys_addr << std::dec << std::endl; + + auto r = dsp_client.process("C7X_MSG_ISTFT_SYNTHESIZE", params); + if (!r.success) + throw PipelineError{"ISTFT batch " + std::to_string(batch_idx + 1) + + " failed: " + r.error_message}; + + dma_buf4.begin_cpu_access(); + const auto* out_ptr = dma_buf4.data(); + + if (debug) { + std::cout << "[App] Frame | InRMS OutRMS | In[0..4] | Out[0..4]" << std::endl; + const size_t available_frames = std::min( + frames_this_batch, real_samples_remaining / ISTFT_HOP_SIZE); + for (size_t f = 0; f < std::min(size_t{4}, available_frames); ++f) { + const size_t in_off = audio_offset + f * ISTFT_HOP_SIZE; + const size_t out_off = f * ISTFT_HOP_SIZE; + float in_sum = 0.0f, out_sum = 0.0f; + for (size_t i = 0; i < ISTFT_HOP_SIZE; i++) { + const float s = static_cast(audio_data[in_off + i]) / 32768.0f; + const float o = static_cast(out_ptr[out_off + i]) / 32768.0f; + in_sum += s * s; + out_sum += o * o; + } + std::cout << "[App] " << std::setw(5) + << (chunk_frame_offset + batch_idx * ISTFT_BATCH_N + f + 1) + << " | " << std::fixed << std::setprecision(4) + << std::sqrt(in_sum / HOP_SIZE) << " " + << std::sqrt(out_sum / HOP_SIZE) + << " | In:"; + for (size_t i = 0; i < 5; i++) + std::cout << std::setw(6) << audio_data[in_off + i] << (i<4?",":""); + std::cout << " | Out:"; + for (size_t i = 0; i < 5; i++) + std::cout << std::setw(6) << out_ptr[out_off + i] << (i<4?",":""); + std::cout << std::endl; + } + } + + size_t samples_to_collect = std::min(samples_this_batch, real_samples_remaining); + std::copy_n(out_ptr, samples_to_collect, + std::back_inserter(processed_audio_data)); + real_samples_remaining -= samples_to_collect; + + dma_buf4.end_cpu_access(); + audio_stream.send_frame(1, out_ptr, samples_to_collect * sizeof(int16_t)); + } + double t_istft_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_istft_start).count() / 1000.0; + + double t_chunk_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_chunk_start).count() / 1000.0; + + std::cout << "[App] Chunk " << (chunk_idx+1) << "/" << num_chunks + << " [" << real_frames_chunk << " real frames" + << (real_frames_chunk < ISTFT_TOTAL_FRAMES ? " + zero-pad" : "") << "]" + << " | STFT=" << std::fixed << std::setprecision(1) << t_stft_ms << "ms" + << " TVM=" << t_tvm_ms << "ms" + << " ISTFT=" << t_istft_ms << "ms" + << " total=" << t_chunk_ms << "ms" << std::endl; + } + + double t_total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_total_start).count() / 1000.0; + + std::cout << "[App] All chunks done | total=" << std::fixed << std::setprecision(1) + << t_total_ms << "ms | " << (processed_audio_data.size() / ISTFT_HOP_SIZE) + << " output frames (" << processed_audio_data.size() << " samples)" << std::endl; + + if (processed_audio_data.size() < original_sample_count) + throw PipelineError{"Pipeline produced fewer samples than expected"}; + processed_audio_data.resize(original_sample_count); + + std::string output_filename = "processed_output.wav"; + if (!saveAudioFile(output_filename, processed_audio_data)) + throw PipelineError{"Failed to save output file"}; + std::cout << "[App] Saved to " << output_filename << std::endl; + return PipelineManager::CommandResult::SUCCESS; + } catch (const std::exception& error) { + std::cerr << "[App] Pipeline failed: " << error.what() << std::endl; + return PipelineManager::CommandResult::ERROR; + } +} diff --git a/example/edge-ai/src/audio_utils.cpp b/example/edge-ai/src/audio_utils.cpp new file mode 100644 index 0000000..621dafc --- /dev/null +++ b/example/edge-ai/src/audio_utils.cpp @@ -0,0 +1,101 @@ +#include "audio_utils.h" +#include +#include +#include + +bool loadAudioFile(const std::string& filename, std::vector& audio_data) +{ + SF_INFO sfinfo{}; + + std::unique_ptr infile{ + sf_open(filename.c_str(), SFM_READ, &sfinfo), &sf_close}; + if (!infile) { + std::cout << "[App] Error: Failed to open audio file: " << filename << std::endl; + return false; + } + + // Validate audio format + if (sfinfo.channels != 1) { + std::cout << "[App] Error: Audio must be mono (1 channel), got " << sfinfo.channels << " channels" << std::endl; + return false; + } + + if (sfinfo.frames <= 0) { + std::cout << "[App] Error: Audio file contains no samples" << std::endl; + return false; + } + + if (sfinfo.samplerate != 16000) { + std::cout << "[App] Error: Audio sample rate is " << sfinfo.samplerate + << "Hz; this pipeline requires 16kHz" << std::endl; + return false; + } + + std::cout << "[App] Audio file info: " << (sfinfo.frames / 160) << " GCRN frames (" + << sfinfo.frames << " samples), " + << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; + + // Read all audio data + audio_data.resize(sfinfo.frames); + const sf_count_t frames_read = sf_readf_short( + infile.get(), audio_data.data(), sfinfo.frames); + + if (frames_read < 0) { + std::cout << "[App] Error: Failed while reading audio data" << std::endl; + return false; + } + if (frames_read != sfinfo.frames) { + std::cout << "[App] Warning: Read " << frames_read << " frames, expected " << sfinfo.frames << std::endl; + audio_data.resize(frames_read); + } + + std::cout << "[App] Loaded " << audio_data.size() << " audio samples (" + << (static_cast(audio_data.size()) / sfinfo.samplerate) + << " seconds)" << std::endl; + + return true; +} + +bool saveAudioFile(const std::string& filename, const std::vector& audio_data) +{ + if (audio_data.empty()) { + std::cout << "[App] Error: Refusing to write an empty audio file" << std::endl; + return false; + } + SF_INFO sfinfo{}; + + // Set output file parameters to match our audio format + sfinfo.samplerate = 16000; // 16kHz + sfinfo.channels = 1; // mono + sfinfo.format = SF_FORMAT_WAV | SF_FORMAT_PCM_16; // WAV file with 16-bit PCM + + std::unique_ptr outfile{ + sf_open(filename.c_str(), SFM_WRITE, &sfinfo), &sf_close}; + if (!outfile) { + std::cout << "[App] Error: Failed to create output audio file: " << filename << std::endl; + std::cout << "[App] Error details: " << sf_strerror(nullptr) << std::endl; + return false; + } + + std::cout << "[App] Saving audio to: " << filename << std::endl; + std::cout << "[App] Output file info: " << (audio_data.size() / 160) << " GCRN frames (" + << audio_data.size() << " samples), " + << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; + + // Write audio data to file + const sf_count_t frames_written = sf_writef_short( + outfile.get(), audio_data.data(), static_cast(audio_data.size())); + + if (frames_written != static_cast(audio_data.size())) { + std::cout << "[App] Warning: Wrote " << frames_written << " frames, expected " << audio_data.size() << std::endl; + return false; + } + + std::cout << "[App] Successfully saved " << (frames_written / 160) << " GCRN frames (" + << frames_written << " samples) to " << filename << std::endl; + std::cout << "[App] Duration: " + << (static_cast(frames_written) / sfinfo.samplerate) + << " seconds" << std::endl; + + return true; +} diff --git a/example/edge-ai/src/generic_task_client.cpp b/example/edge-ai/src/dsp_task_client.cpp similarity index 89% rename from example/edge-ai/src/generic_task_client.cpp rename to example/edge-ai/src/dsp_task_client.cpp index 660fd40..7158222 100644 --- a/example/edge-ai/src/generic_task_client.cpp +++ b/example/edge-ai/src/dsp_task_client.cpp @@ -1,4 +1,4 @@ -#include "generic_task_client.h" +#include "dsp_task_client.h" #include #include #include @@ -11,10 +11,6 @@ extern "C" { namespace { -constexpr int C7_PROC_ID = 8; -constexpr int RMT_EP = 13; -constexpr uint32_t TVM_STAGING_PHYS = 0xa3000000U; -constexpr uint32_t TVM_RESULT_PHYS = 0xabc00000U; uint32_t parameter_value(const std::map& parameters, const std::string& name, uint32_t default_value, @@ -90,19 +86,18 @@ bool exchange_message(int descriptor, Message& request, Message& response) } // namespace -GenericTaskClient::GenericTaskClient() - : rpmsg_fd_(-1), initialized_(false), sequence_number_(1) +DspTaskClient::DspTaskClient() + : rpmsg_fd_(-1), proc_id_(0), endpoint_(0), initialized_(false), sequence_number_(1) { - shared_input_addr_ = TVM_STAGING_PHYS; - shared_output_addr_ = TVM_RESULT_PHYS; } -GenericTaskClient::~GenericTaskClient() +DspTaskClient::~DspTaskClient() { shutdown(); } -bool GenericTaskClient::initialize(uint32_t max_input_size, uint32_t max_output_size) +bool DspTaskClient::initialize(int proc_id, int endpoint, + uint32_t max_input_size, uint32_t max_output_size) { (void)max_input_size; (void)max_output_size; @@ -110,6 +105,9 @@ bool GenericTaskClient::initialize(uint32_t max_input_size, uint32_t max_output_ return true; } + proc_id_ = proc_id; + endpoint_ = endpoint; + if (!open_rpmsg_device()) { return false; } @@ -118,16 +116,16 @@ bool GenericTaskClient::initialize(uint32_t max_input_size, uint32_t max_output_ return true; } -bool GenericTaskClient::open_rpmsg_device() +bool DspTaskClient::open_rpmsg_device() { - rpmsg_fd_ = init_rpmsg(C7_PROC_ID, RMT_EP); + rpmsg_fd_ = init_rpmsg(proc_id_, endpoint_); if (rpmsg_fd_ < 0) { return false; } return true; } -void GenericTaskClient::close_rpmsg_device() +void DspTaskClient::close_rpmsg_device() { if (rpmsg_fd_ >= 0) { ::close(rpmsg_fd_); @@ -135,7 +133,7 @@ void GenericTaskClient::close_rpmsg_device() } } -GenericTaskClient::ProcessingResult GenericTaskClient::process(const std::string& message_type, +DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& message_type, void* input_data, uint32_t input_size, void* output_data, @@ -164,8 +162,8 @@ GenericTaskClient::ProcessingResult GenericTaskClient::process(const std::string req.hdr.len = sizeof(struct stft_process_msg); req.hdr.status = 0; - req.input_buffer = parameter_value(parameters, "input_buffer", shared_input_addr_, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", shared_output_addr_, 16); + req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); + req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); req.input_frame = parameter_value(parameters, "input_frame", 0); req.output_frame = parameter_value(parameters, "output_frame", 0); req.graph_id = parameter_value(parameters, "graph_id", 0); @@ -209,8 +207,8 @@ GenericTaskClient::ProcessingResult GenericTaskClient::process(const std::string req.hdr.len = sizeof(struct stft_process_msg); req.hdr.status = 0; - req.input_buffer = parameter_value(parameters, "input_buffer", shared_input_addr_, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", shared_output_addr_, 16); + req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); + req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); req.input_frame = parameter_value(parameters, "input_frame", 0); req.output_frame = parameter_value(parameters, "output_frame", 0); req.graph_id = parameter_value(parameters, "graph_id", 0); @@ -254,8 +252,8 @@ GenericTaskClient::ProcessingResult GenericTaskClient::process(const std::string req.hdr.len = sizeof(struct deinterleave_interleave_msg); req.hdr.status = 0; - req.input_buffer = parameter_value(parameters, "input_buffer", shared_input_addr_, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", shared_output_addr_, 16); + req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); + req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); req.input_frame = parameter_value(parameters, "input_frame", 0); req.fft_size = parameter_value(parameters, "fft_size", 0); req.flag = parameter_value(parameters, "flag", 0); @@ -292,14 +290,14 @@ GenericTaskClient::ProcessingResult GenericTaskClient::process(const std::string return result; } -GenericTaskClient::ProcessingResult GenericTaskClient::process( +DspTaskClient::ProcessingResult DspTaskClient::process( const std::string& message_type, const std::map& parameters) { return process(message_type, nullptr, 0, nullptr, 0, parameters); } -GenericTaskClient::ProcessingResult GenericTaskClient::get_service_status() +DspTaskClient::ProcessingResult DspTaskClient::get_service_status() { ProcessingResult result = {}; result.success = initialized_; @@ -309,13 +307,13 @@ GenericTaskClient::ProcessingResult GenericTaskClient::get_service_status() return result; } -bool GenericTaskClient::ping_service() +bool DspTaskClient::ping_service() { // STFT service doesn't have separate ping - just return initialized status return initialized_; } -void GenericTaskClient::shutdown() +void DspTaskClient::shutdown() { if (initialized_) { close_rpmsg_device(); @@ -323,7 +321,7 @@ void GenericTaskClient::shutdown() } } -std::string GenericTaskClient::get_error_string(int32_t error_code) +std::string DspTaskClient::get_error_string(int32_t error_code) { switch (error_code) { case C7X_STATUS_SUCCESS: return "Success"; diff --git a/example/edge-ai/src/main.cpp b/example/edge-ai/src/main.cpp index bedc543..fee8d30 100644 --- a/example/edge-ai/src/main.cpp +++ b/example/edge-ai/src/main.cpp @@ -8,7 +8,7 @@ namespace { -constexpr std::string_view APP_VERSION = "0.0.3"; +constexpr std::string_view APP_VERSION = "0.0.4"; constexpr std::string_view BUILD_DATE = __DATE__; constexpr std::string_view BUILD_TIME = __TIME__; @@ -33,10 +33,9 @@ void print_usage(std::string_view program) { std::cout << "Usage:\n" - << " " << program << " Interactive mode\n" << " " << program << " Run a JSON pipeline\n" << " " << program << " --debug Enable per-batch logs\n" - << " " << program << " --version Show version and build info\n" + << " " << program << " --version Show version and build info\n" << " " << program << " --help Show this help\n\n" << "Examples:\n" << " " << program << " pipeline_tvm_inference.json\n" @@ -57,7 +56,7 @@ int main(int argc, char* argv[]) print_usage(argv[0]); return EXIT_SUCCESS; } - if (argument == "--version" || argument == "-v") { + if (argument == "--version" || argument == "-v") { print_version(); return EXIT_SUCCESS; } @@ -77,6 +76,12 @@ int main(int argc, char* argv[]) json_file = argument; } + if (json_file.empty()) { + std::cerr << "[App] Error: A pipeline JSON file is required\n"; + print_usage(argv[0]); + return EXIT_FAILURE; + } + setup_signal_handlers(); print_version(); std::cout << "===========================================\n" @@ -85,9 +90,7 @@ int main(int argc, char* argv[]) PipelineManager application; application.set_debug(debug); - const int exit_code = json_file.empty() - ? application.run() - : application.run_from_json_file(json_file); + const int exit_code = application.run_from_json_file(json_file); std::cout << "[App] Application exited with code " << exit_code << '\n'; return exit_code; } catch (const std::exception& error) { diff --git a/example/edge-ai/src/pipeline_common.cpp b/example/edge-ai/src/pipeline_common.cpp new file mode 100644 index 0000000..6e14a8e --- /dev/null +++ b/example/edge-ai/src/pipeline_common.cpp @@ -0,0 +1,116 @@ +#include "pipeline_common.h" +#include + +std::string hex_address(uint64_t address) +{ + std::ostringstream value; + value << "0x" << std::hex << address; + return value.str(); +} + +DmaBuffer::DmaBuffer(size_t bytes, std::string_view purpose) +{ + if (bytes > std::numeric_limits::max()) + throw PipelineError{"DMA allocation is larger than the API limit"}; + char heap[] = "linux,cma"; + char remoteproc[] = "/dev/remoteproc0"; + if (dmabuf_heap_init(heap, static_cast(bytes), remoteproc, ¶ms_) != 0) + throw PipelineError{"Failed to allocate DMA buffer for " + std::string{purpose}}; + allocated_ = true; +} + +DmaBuffer::~DmaBuffer() +{ + if (allocated_) dmabuf_heap_destroy(¶ms_); +} + +void DmaBuffer::begin_cpu_access() const { sync(DMA_BUF_SYNC_START); } +void DmaBuffer::end_cpu_access() const { sync(DMA_BUF_SYNC_END); } + +void DmaBuffer::sync(int operation) const +{ + if (dmabuf_sync(params_.dma_buf_fd, operation) != 0) + throw PipelineError{"DMA buffer synchronization failed"}; +} + +AudioStream::AudioStream() noexcept { open(); } + +AudioStream::~AudioStream() +{ + close_fd(client_); + close_fd(server_); + ::unlink(socket_path_.data()); +} + +void AudioStream::send_frame(uint8_t direction, const void* pcm, size_t bytes) noexcept +{ + if (server_ < 0 || !pcm || bytes > std::numeric_limits::max()) + return; + if (client_ < 0) + client_ = ::accept(server_, nullptr, nullptr); + if (client_ < 0) + return; + const auto header = make_header(direction, static_cast(bytes)); + if (!send_all(header.data(), header.size()) || !send_all(pcm, bytes)) + close_fd(client_); +} + +std::array AudioStream::make_header(uint8_t direction, + uint32_t pcm_bytes) noexcept +{ + std::array header{}; + header[0] = std::byte{'E'}; + header[1] = std::byte{'A'}; + header[2] = std::byte{'S'}; + header[3] = std::byte{'P'}; + header[4] = static_cast(direction); + write_u32_le(header, 5, 16000); + write_u32_le(header, 9, pcm_bytes); + return header; +} + +void AudioStream::write_u32_le(std::array& destination, + size_t offset, uint32_t value) noexcept +{ + for (size_t index = 0; index < sizeof(value); ++index) + destination[offset + index] = + static_cast((value >> (index * 8)) & 0xffU); +} + +void AudioStream::open() noexcept +{ + ::unlink(socket_path_.data()); + server_ = ::socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (server_ < 0) + return; + sockaddr_un address{}; + address.sun_family = AF_UNIX; + std::copy(socket_path_.begin(), socket_path_.end(), address.sun_path); + if (::bind(server_, reinterpret_cast(&address), sizeof(address)) != 0 || + ::listen(server_, 1) != 0) + close_fd(server_); +} + +bool AudioStream::send_all(const void* data, size_t bytes) noexcept +{ + const auto* cursor = static_cast(data); + size_t sent = 0; + while (sent < bytes) { + const auto count = ::send(client_, cursor + sent, bytes - sent, MSG_NOSIGNAL); + if (count > 0) { + sent += static_cast(count); + } else if (count < 0 && errno == EINTR) { + continue; + } else { + return false; + } + } + return true; +} + +void AudioStream::close_fd(int& descriptor) noexcept +{ + if (descriptor >= 0) + ::close(descriptor); + descriptor = -1; +} diff --git a/example/edge-ai/src/pipeline_manager.cpp b/example/edge-ai/src/pipeline_manager.cpp index c044c51..e3e8705 100644 --- a/example/edge-ai/src/pipeline_manager.cpp +++ b/example/edge-ai/src/pipeline_manager.cpp @@ -1,209 +1,24 @@ #include "pipeline_manager.h" +#include "pipeline_common.h" +#include "audio_utils.h" +#include "tvm_pipeline.h" +#include "stft_istft_pipeline.h" +#include "audio_enhancement_pipeline.h" #include -#include #include #include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include extern "C" { -#include "fw_loader.h" -#include -#include +#include } -namespace { - -// Pipeline JSON file paths for --mode command-line invocation -static const char* PIPELINE_FILE_STFT_ISTFT = "json_files/pipeline_stft_istft.json"; -static const char* PIPELINE_FILE_TVM_ONLY = "json_files/pipeline_tvm_inference.json"; -static const char* PIPELINE_FILE_FULL = "json_files/pipeline_audio_enhancement.json"; - -class PipelineError : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -std::string hex_address(uint64_t address) -{ - std::ostringstream value; - value << "0x" << std::hex << address; - return value.str(); -} - -class DmaBuffer { -public: - DmaBuffer(size_t bytes, std::string_view purpose) - { - if (bytes > std::numeric_limits::max()) - throw PipelineError{"DMA allocation is larger than the API limit"}; - char heap[] = "linux,cma"; - char remoteproc[] = "/dev/remoteproc0"; - if (dmabuf_heap_init(heap, static_cast(bytes), remoteproc, ¶ms_) != 0) - throw PipelineError{"Failed to allocate DMA buffer for " + std::string{purpose}}; - allocated_ = true; - } - - ~DmaBuffer() { if (allocated_) dmabuf_heap_destroy(¶ms_); } - DmaBuffer(const DmaBuffer&) = delete; - DmaBuffer& operator=(const DmaBuffer&) = delete; - - dma_buf_params* operator->() noexcept { return ¶ms_; } - const dma_buf_params* operator->() const noexcept { return ¶ms_; } - - template - T* data() noexcept { return reinterpret_cast(params_.kern_addr); } - - void begin_cpu_access() const { sync(DMA_BUF_SYNC_START); } - void end_cpu_access() const { sync(DMA_BUF_SYNC_END); } - -private: - void sync(int operation) const - { - if (dmabuf_sync(params_.dma_buf_fd, operation) != 0) - throw PipelineError{"DMA buffer synchronization failed"}; - } - - dma_buf_params params_{}; - bool allocated_{false}; -}; - -class AudioStream { -public: - AudioStream() noexcept { open(); } - ~AudioStream() { close_fd(client_); close_fd(server_); ::unlink(socket_path_.data()); } - AudioStream(const AudioStream&) = delete; - AudioStream& operator=(const AudioStream&) = delete; - - void send_frame(uint8_t direction, const void* pcm, size_t bytes) noexcept - { - if (server_ < 0 || !pcm || bytes > std::numeric_limits::max()) - return; - if (client_ < 0) - client_ = ::accept(server_, nullptr, nullptr); - if (client_ < 0) - return; - const auto header = make_header(direction, static_cast(bytes)); - if (!send_all(header.data(), header.size()) || !send_all(pcm, bytes)) - close_fd(client_); - } - -private: - static std::array make_header(uint8_t direction, - uint32_t pcm_bytes) noexcept - { - std::array header{}; - header[0] = std::byte{'E'}; - header[1] = std::byte{'A'}; - header[2] = std::byte{'S'}; - header[3] = std::byte{'P'}; - header[4] = static_cast(direction); - write_u32_le(header, 5, 16000); - write_u32_le(header, 9, pcm_bytes); - return header; - } - - static void write_u32_le(std::array& destination, - size_t offset, uint32_t value) noexcept - { - for (size_t index = 0; index < sizeof(value); ++index) - destination[offset + index] = - static_cast((value >> (index * 8)) & 0xffU); - } - - void open() noexcept - { - ::unlink(socket_path_.data()); - server_ = ::socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); - if (server_ < 0) - return; - sockaddr_un address{}; - address.sun_family = AF_UNIX; - std::copy(socket_path_.begin(), socket_path_.end(), address.sun_path); - if (::bind(server_, reinterpret_cast(&address), sizeof(address)) != 0 || - ::listen(server_, 1) != 0) - close_fd(server_); - } - - bool send_all(const void* data, size_t bytes) noexcept - { - const auto* cursor = static_cast(data); - size_t sent = 0; - while (sent < bytes) { - const auto count = ::send(client_, cursor + sent, bytes - sent, MSG_NOSIGNAL); - if (count > 0) { - sent += static_cast(count); - } else if (count < 0 && errno == EINTR) { - continue; - } else { - return false; - } - } - return true; - } - - static void close_fd(int& descriptor) noexcept - { - if (descriptor >= 0) - ::close(descriptor); - descriptor = -1; - } - - static constexpr std::string_view socket_path_{"/tmp/edge-ai-speech.sock"}; - static_assert(socket_path_.size() < sizeof(sockaddr_un{}.sun_path), - "Audio stream socket path is too long"); - int server_{-1}; - int client_{-1}; -}; - -} // namespace - -void validate_pipeline_files() { - const std::vector pipeline_files = { - PIPELINE_FILE_STFT_ISTFT, - PIPELINE_FILE_TVM_ONLY, - PIPELINE_FILE_FULL - }; - - for (const auto& file_path : pipeline_files) { - // Try to open the file for reading - if (!std::filesystem::is_regular_file(file_path)) - throw PipelineError{"Required pipeline file is missing: " + std::string{file_path}}; - } -} - -// Helper function to load JSON from file -static std::string load_json_file(const std::string& file_path); - - PipelineManager::PipelineManager() - : initialized_(false), app_name_("edge-ai") + : initialized_(false) { state_.artifacts_loaded = false; state_.tvm_artifacts_configured = false; state_.input_configured = false; - state_.current_pipeline_file = ""; - state_.current_input_file = ""; - state_.tvm_staging_buffer = nullptr; - state_.tvm_result_buffer = nullptr; - state_.staging_buffer_size = 0; - state_.result_buffer_size = 0; } PipelineManager::~PipelineManager() @@ -212,117 +27,12 @@ PipelineManager::~PipelineManager() bool PipelineManager::initialize() { - tvm_client_ = std::make_shared(); - generic_client_ = std::make_unique(); - - // TVM client will be initialized later when pipeline is loaded (needs artifacts path from config) - - initializeCommands(); + tvm_client_ = std::make_shared(); + generic_client_ = std::make_unique(); initialized_ = true; return true; } -int PipelineManager::run() -{ - if (!initialize()) { - std::cout << "[App] Failed to initialize application" << std::endl; - return -1; - } - - printWelcome(); - - while (true) { - std::string prompt = getCurrentPrompt(); - std::unique_ptr input_line{ - readline(prompt.c_str()), &std::free}; - - if (!input_line) { - std::cout << std::endl << "[App] EOF received, exiting..." << std::endl; - break; - } - - std::string line(input_line.get()); - - if (!line.empty() && line.find_first_not_of(" \t") != std::string::npos) { - add_history(input_line.get()); - } - - if (line.empty() || line.find_first_not_of(" \t") == std::string::npos) { - continue; - } - - auto tokens = parseCommand(line); - if (tokens.empty()) { - continue; - } - - CommandResult result = executeCommand(tokens); - if (result == CommandResult::QUIT) { - break; - } - } - - return 0; -} - -// Helper function implementation -static std::string load_json_file(const std::string& file_path) -{ - std::ifstream file(file_path); - if (!file.is_open()) { - std::cerr << "[App] Error: Cannot open pipeline file: " << file_path << std::endl; - return ""; - } - - std::string json_content((std::istreambuf_iterator(file)), - std::istreambuf_iterator()); - return json_content; -} - -int PipelineManager::run_direct(PipelineMode mode, const std::string& input_file, - const std::string& artifacts_path) -{ - if (!initialize()) { - std::cout << "[App] Failed to initialize application" << std::endl; - return -1; - } - - std::string json_file; - switch (mode) { - case PipelineMode::STFT_ISTFT: json_file = PIPELINE_FILE_STFT_ISTFT; break; - case PipelineMode::TVM_ONLY: json_file = PIPELINE_FILE_TVM_ONLY; break; - case PipelineMode::FULL: json_file = PIPELINE_FILE_FULL; break; - } - - std::string json_content = load_json_file(json_file); - if (json_content.empty()) { - std::cout << "[App] Error: Failed to read pipeline file: " << json_file << std::endl; - std::cout << "[App] Hint: Make sure json_files directory exists in the working directory" << std::endl; - return -1; - } - - if (!loadPipelineFromJson(json_content)) { - std::cout << "[App] Error: Failed to load pipeline configuration" << std::endl; - return -1; - } - - std::cout << "[App] Pipeline: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "[App] Description: " << state_.pipeline_config.description << std::endl; - - if (!artifacts_path.empty()) { - if (handleTvmArtifacts({artifacts_path}) != CommandResult::SUCCESS) { - return -1; - } - } - - if (handleInput({input_file}) != CommandResult::SUCCESS) { - return -1; - } - - CommandResult result = handleRun({}); - return (result == CommandResult::SUCCESS) ? 0 : 1; -} - int PipelineManager::run_from_json_file(const std::string& json_file_path) { if (!initialize()) { @@ -345,159 +55,64 @@ int PipelineManager::run_from_json_file(const std::string& json_file_path) state_.current_pipeline_file = json_file_path; - std::cout << "[App] Pipeline: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "[App] Description: " << state_.pipeline_config.description << std::endl; + std::cout << "[App] Pipeline type: " << state_.pipeline_config.pipeline_type << std::endl; + std::cout << "[App] Description: " << state_.pipeline_config.description << std::endl; + // Configure TVM artifacts if specified in JSON if (!state_.pipeline_config.artifacts_path.empty()) { - if (handleTvmArtifacts({state_.pipeline_config.artifacts_path}) != CommandResult::SUCCESS) { + const std::string& path = state_.pipeline_config.artifacts_path; + if (!std::filesystem::exists(path)) { + std::cout << "[App] Error: Artifacts path not found: " << path << std::endl; return -1; } + state_.tvm_artifacts_paths = {path}; + state_.tvm_artifacts_configured = true; + std::cout << "[App] TVM artifacts configured: " << path << std::endl; } + // Configure input file if (state_.pipeline_config.input_file.empty()) { std::cout << "[App] Error: No input_file specified in pipeline JSON" << std::endl; return -1; } - - if (handleInput({state_.pipeline_config.input_file}) != CommandResult::SUCCESS) { + const std::string& input_file = state_.pipeline_config.input_file; + if (!std::filesystem::exists(input_file)) { + std::cout << "[App] Error: Input file not found: " << input_file << std::endl; return -1; } - - CommandResult result = handleRun({}); - return (result == CommandResult::SUCCESS) ? 0 : 1; -} - -void PipelineManager::initializeCommands() -{ - registerCommand("help", "Show available commands", - {"help"}, - [this](const auto& args) { return handleHelp(args); }); - - registerCommand("pipeline", "Load pipeline configuration from JSON file", - {"pipeline ", "pipeline sample.json"}, - [this](const auto& args) { return handlePipeline(args); }); - - registerCommand("tvm_artifacts", "Configure TVM model artifacts", - {"tvm_artifacts [file2.so] ...", "tvm_artifacts model.so"}, - [this](const auto& args) { return handleTvmArtifacts(args); }); - - registerCommand("input", "Set input data for pipeline execution", - {"input ", "input sample.wav"}, - [this](const auto& args) { return handleInput(args); }); - - registerCommand("show_pipeline", "Display current pipeline structure", - {"show_pipeline"}, - [this](const auto& args) { return handleShowPipeline(args); }); - - registerCommand("run", "Execute pipeline (loads input and artifacts as needed)", - {"run"}, - [this](const auto& args) { return handleRun(args); }); - - registerCommand("status", "Show current application state", - {"status"}, - [this](const auto& args) { return handleStatus(args); }); - - registerCommand("quit", "Exit the application", - {"quit", "exit"}, - [this](const auto& args) { return handleQuit(args); }); -} - -void PipelineManager::printWelcome() -{ - std::cout << "\n===========================================\n"; - std::cout << " RPMsg Inference Example\n"; - std::cout << "===========================================\n"; - std::cout << "Type 'help' for available commands\n" << std::endl; -} - -void PipelineManager::printPrompt() -{ - std::cout << getCurrentPrompt(); -} - -std::vector PipelineManager::parseCommand(const std::string& input) -{ - std::vector tokens; - std::istringstream iss(input); - std::string token; - - while (iss >> token) { - tokens.push_back(token); + const auto extension = std::filesystem::path{input_file}.extension(); + if (extension == ".wav") + state_.input_type = InputType::AUDIO_WAV; + else if (extension == ".bin") + state_.input_type = InputType::TENSOR_BIN; + else { + std::cout << "[App] Warning: Unknown input type for file: " << input_file << std::endl; + state_.input_type = InputType::UNKNOWN; } + state_.current_input_file = input_file; + state_.input_configured = true; + std::cout << "[App] Input configured: " << input_file << std::endl; - return tokens; -} - -PipelineManager::CommandResult PipelineManager::executeCommand(const std::vector& tokens) -{ - if (tokens.empty()) { - return CommandResult::SUCCESS; - } + if (!validateConfiguration()) + return -1; - std::string command = tokens[0]; - std::vector args(tokens.begin() + 1, tokens.end()); + std::cout << "[App] Stages: " << state_.pipeline_config.stages.size() << std::endl; - if (command == "exit") { - command = "quit"; - } + const auto& pipeline_type = state_.pipeline_config.pipeline_type; + CommandResult result; - auto it = commands_.find(command); - if (it != commands_.end()) { - return it->second.handler(args); + if (pipeline_type == "tvm_only") { + result = run_tvm_pipeline(state_, *tvm_client_); + } else if (pipeline_type == "audio_enhancement") { + result = run_audio_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "stft_istft") { + result = run_stft_istft_pipeline(state_, *generic_client_, debug_); } else { - std::cout << "[App] Unknown command: " << command << std::endl; - std::cout << "[App] Type 'help' for available commands" << std::endl; - return CommandResult::ERROR; - } -} - -PipelineManager::CommandResult PipelineManager::handleHelp(const std::vector&) -{ - std::cout << "\nAvailable commands:\n"; - std::cout << "===================\n"; - - for (const auto& [name, info] : commands_) { - std::cout << name << " - " << info.description << "\n"; - std::cout << "Examples: "; - for (size_t i = 0; i < info.examples.size(); ++i) { - std::cout << info.examples[i]; - if (i < info.examples.size() - 1) std::cout << ", "; - } - std::cout << "\n\n"; - } - - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::handlePipeline(const std::vector& args) -{ - if (args.empty()) { - std::cout << "[App] Error: Pipeline file required" << std::endl; - std::cout << "Usage: pipeline " << std::endl; - return CommandResult::ERROR; - } - - std::string pipeline_file = args[0]; - - std::ifstream file(pipeline_file); - if (!file.is_open()) { - std::cout << "[App] Error: Pipeline file not found: " << pipeline_file << std::endl; - return CommandResult::ERROR; - } - - std::string json_content((std::istreambuf_iterator(file)), - std::istreambuf_iterator()); - if (!loadPipelineFromJson(json_content)) { - return CommandResult::ERROR; + std::cout << "[App] Error: Unknown pipeline_type: " << pipeline_type << std::endl; + return -1; } - state_.current_pipeline_file = pipeline_file; - - std::cout << "[App] Pipeline configuration loaded: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "[App] Description: " << state_.pipeline_config.description << std::endl; - std::cout << "[App] Stages: " << state_.pipeline_config.stages.size() << std::endl; - - return CommandResult::SUCCESS; + return (result == CommandResult::SUCCESS) ? 0 : 1; } bool PipelineManager::loadPipelineFromJson(const std::string& json_content) @@ -521,14 +136,27 @@ bool PipelineManager::loadPipelineFromJson(const std::string& json_content) }; PipelineConfig config; - if (!read_string(root.get(), "pipeline_id", config.pipeline_id, true) || - !read_string(root.get(), "description", config.description, false) || - !read_string(root.get(), "input_file", config.input_file, true) || - !read_string(root.get(), "artifacts_path", config.artifacts_path, false)) { - std::cout << "[App] Error: Missing or invalid pipeline string field" << std::endl; + if (!read_string(root.get(), "pipeline_type", config.pipeline_type, true) || + !read_string(root.get(), "description", config.description, false) || + !read_string(root.get(), "input_file", config.input_file, true) || + !read_string(root.get(), "artifacts_path",config.artifacts_path, false)) { + std::cout << "[App] Error: Missing or invalid pipeline field" << std::endl; return false; } + // Parse dsp_config — required for pipelines using DSP generic service + json_object* dsp_cfg = nullptr; + if (json_object_object_get_ex(root.get(), "dsp_config", &dsp_cfg) && + json_object_is_type(dsp_cfg, json_type_object)) { + json_object* val = nullptr; + if (json_object_object_get_ex(dsp_cfg, "proc_id", &val) && + json_object_is_type(val, json_type_int)) + config.dsp_config.proc_id = json_object_get_int(val); + if (json_object_object_get_ex(dsp_cfg, "endpoint", &val) && + json_object_is_type(val, json_type_int)) + config.dsp_config.endpoint = json_object_get_int(val); + } + json_object* stages = nullptr; if (!json_object_object_get_ex(root.get(), "stages", &stages) || !json_object_is_type(stages, json_type_array) || @@ -543,8 +171,8 @@ bool PipelineManager::loadPipelineFromJson(const std::string& json_content) json_object* object = json_object_array_get_idx(stages, index); PipelineStage stage; if (!object || !json_object_is_type(object, json_type_object) || - !read_string(object, "stage_id", stage.stage_id, true) || - !read_string(object, "service", stage.service, true) || + !read_string(object, "stage_id", stage.stage_id, true) || + !read_string(object, "service", stage.service, true) || !read_string(object, "message_type", stage.message_type, true) || (stage.service != "generic" && stage.service != "tvm")) { std::cout << "[App] Error: Invalid pipeline stage at index " << index << std::endl; @@ -575,714 +203,25 @@ bool PipelineManager::loadPipelineFromJson(const std::string& json_content) return true; } -PipelineManager::CommandResult PipelineManager::handleTvmArtifacts(const std::vector& args) -{ - if (args.empty()) { - std::cout << "[App] Error: At least one artifact file/directory required" << std::endl; - std::cout << "Usage: tvm_artifacts [file2.so] ..." << std::endl; - return CommandResult::ERROR; - } - - state_.tvm_artifacts_paths.clear(); - - for (const std::string& artifact : args) { - if (!std::filesystem::exists(artifact)) { - std::cout << "[App] Error: Artifact not found: " << artifact << std::endl; - return CommandResult::ERROR; - } - state_.tvm_artifacts_paths.push_back(artifact); - } - - state_.tvm_artifacts_configured = true; - - std::cout << "[App] TVM artifacts configured (" << args.size() << " files):" << std::endl; - for (const std::string& artifact : state_.tvm_artifacts_paths) { - std::cout << "[App] " << artifact << std::endl; - } - - std::cout << "[App] TVM artifacts path set (will be used when TVM stage is enabled)" << std::endl; - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::handleInput(const std::vector& args) -{ - if (args.empty()) { - std::cout << "[App] Error: Input file required" << std::endl; - std::cout << "Usage: input " << std::endl; - return CommandResult::ERROR; - } - - std::string input_file = args[0]; - - std::ifstream file(input_file); - if (!file.is_open()) { - std::cout << "[App] Error: Input file not found: " << input_file << std::endl; - return CommandResult::ERROR; - } - const auto extension = std::filesystem::path{input_file}.extension(); - if (extension == ".wav") { - state_.input_type = InputType::AUDIO_WAV; - } else if (extension == ".bin") { - state_.input_type = InputType::TENSOR_BIN; - } else { - std::cout << "[App] Warning: Unknown input type for file: " << input_file << std::endl; - state_.input_type = InputType::UNKNOWN; - } - - state_.current_input_file = input_file; - state_.input_configured = true; - - std::string type_str; - switch (state_.input_type) { - case InputType::AUDIO_WAV: type_str = "WAV audio"; break; - case InputType::TENSOR_BIN: type_str = "BIN tensor"; break; - default: type_str = "unknown"; break; - } - - std::cout << "[App] Input configured: " << input_file << " (type: " << type_str << ")" << std::endl; - - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::handleShowPipeline(const std::vector&) -{ - if (!state_.pipeline_config.loaded) { - std::cout << "[App] No pipeline configuration loaded" << std::endl; - std::cout << "Use 'pipeline ' to load a pipeline configuration" << std::endl; - return CommandResult::SUCCESS; - } - - std::cout << std::endl << "Pipeline Configuration:" << std::endl; - std::cout << "======================" << std::endl; - std::cout << "Pipeline ID: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "Description: " << state_.pipeline_config.description << std::endl; - std::cout << "Source File: " << state_.current_pipeline_file << std::endl; - std::cout << std::endl << "Stages (" << state_.pipeline_config.stages.size() << "):" << std::endl; - - for (size_t i = 0; i < state_.pipeline_config.stages.size(); i++) { - const auto& stage = state_.pipeline_config.stages[i]; - std::cout << std::endl << " [" << (i + 1) << "] " << stage.stage_id << " (" << stage.service << " service)" << std::endl; - std::cout << " Message Type: " << stage.message_type << std::endl; - - if (!stage.parameters.empty()) { - std::cout << " Parameters:" << std::endl; - for (const auto& [key, value] : stage.parameters) { - std::cout << " " << key << ": " << value << std::endl; - } - } - } - - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::handleRun(const std::vector&) -{ - if (!validateConfiguration()) { - return CommandResult::ERROR; - } - - std::cout << "[App] Executing sequential pipeline: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "[App] Stages: " << state_.pipeline_config.stages.size() << std::endl; - - return executeSequentialPipeline(); -} - -PipelineManager::CommandResult PipelineManager::handleStatus(const std::vector&) -{ - std::cout << std::endl << "System Status:" << std::endl; - std::cout << "==============" << std::endl; - std::cout << "Application: " << (initialized_ ? "Initialized" : "Not Initialized") << std::endl; - - if (state_.pipeline_config.loaded) { - std::cout << std::endl << "Pipeline Configuration:" << std::endl; - std::cout << "======================" << std::endl; - std::cout << "Pipeline ID: " << state_.pipeline_config.pipeline_id << std::endl; - std::cout << "Description: " << state_.pipeline_config.description << std::endl; - std::cout << "Source File: " << state_.current_pipeline_file << std::endl; - for (size_t i = 0; i < state_.pipeline_config.stages.size(); i++) { - std::cout << " [" << (i + 1) << "] " << state_.pipeline_config.stages[i].stage_id - << " (" << state_.pipeline_config.stages[i].service << ")" << std::endl; - } - } else { - std::cout << "Pipeline Configuration: Not loaded" << std::endl; - } - - std::cout << std::endl << "Configuration Status:" << std::endl; - std::cout << "=====================" << std::endl; - std::cout << "TVM Artifacts: " << (state_.tvm_artifacts_configured ? "Configured" : "Not configured") << std::endl; - if (state_.tvm_artifacts_configured) { - std::cout << " Files: " << state_.tvm_artifacts_paths.size() << std::endl; - } - std::cout << "Input Data: " << (state_.input_configured ? "Configured" : "Not configured") << std::endl; - if (state_.input_configured) { - std::cout << " File: " << state_.current_input_file << std::endl; - } - std::cout << "Artifacts Loaded: " << (state_.artifacts_loaded ? "Yes" : "No") << std::endl; - - std::cout << std::endl << "Readiness Check:" << std::endl; - std::cout << "================" << std::endl; - bool ready = state_.pipeline_config.loaded && state_.tvm_artifacts_configured && state_.input_configured; - std::cout << "Ready to run: " << (ready ? "Yes" : "No") << std::endl; - - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::handleQuit(const std::vector&) -{ - std::cout << "[App] Exiting..." << std::endl; - return CommandResult::QUIT; -} - bool PipelineManager::validateConfiguration() { if (!state_.pipeline_config.loaded) { std::cout << "[App] Error: No pipeline configuration loaded" << std::endl; - std::cout << "Use 'pipeline ' to load a pipeline configuration" << std::endl; return false; } bool has_tvm_stages = false; for (const auto& stage : state_.pipeline_config.stages) { - if (stage.service == "tvm") { - has_tvm_stages = true; - break; - } + if (stage.service == "tvm") { has_tvm_stages = true; break; } } if (has_tvm_stages && !state_.tvm_artifacts_configured) { std::cout << "[App] Error: Pipeline has TVM stages but no TVM artifacts configured" << std::endl; - std::cout << "Use 'tvm_artifacts ...' to configure TVM artifacts" << std::endl; return false; } if (!state_.input_configured) { std::cout << "[App] Error: No input data configured" << std::endl; - std::cout << "Use 'input ' to configure input data" << std::endl; - return false; - } - - return true; -} - -std::string PipelineManager::getCurrentPrompt() -{ - return app_name_ + "> "; -} - -void PipelineManager::registerCommand(const std::string& name, const std::string& description, - const std::vector& examples, - std::function&)> handler) -{ - commands_[name] = {description, examples, handler}; -} - -PipelineManager::CommandResult PipelineManager::executeTensorPipeline() { - std::cout << "\n[App] === Executing Tensor-Only Pipeline ===" << std::endl; - - // Verify we have exactly 1 TVM stage - if (state_.pipeline_config.stages.size() != 1 || - state_.pipeline_config.stages[0].service != "tvm") { - std::cout << "[App] Error: Tensor pipeline must have exactly 1 TVM stage" << std::endl; - return CommandResult::ERROR; - } - - std::cout << "[App] Running TVM inference" << std::endl; - std::cout << "[App] Artifacts: " << state_.tvm_artifacts_paths[0] << std::endl; - std::cout << "[App] Input: " << state_.current_input_file << std::endl; - - if (!tvm_client_->initialize(state_.tvm_artifacts_paths[0])) { - std::cout << "[App] Error: Failed to initialize TVM client" << std::endl; - return CommandResult::ERROR; - } - - if (!tvm_client_->run_inference(state_.current_input_file)) { - std::cout << "[App] Error: TVM inference failed" << std::endl; - return CommandResult::ERROR; - } - - // Save output to .bin file - const std::filesystem::path input_path{state_.current_input_file}; - const std::string output_file = - (input_path.parent_path() / (input_path.stem().string() + "_output.bin")).string(); - - const std::vector& output = tvm_client_->get_output(); - if (!saveTensorFile(output_file, output)) { - std::cout << "[App] Error: Failed to save output tensor" << std::endl; - return CommandResult::ERROR; - } - - std::cout << "[App] Output saved to: " << output_file << std::endl; - std::cout << "[App] Pipeline completed successfully" << std::endl; - - return CommandResult::SUCCESS; -} - -PipelineManager::CommandResult PipelineManager::executeSequentialPipeline() -{ - if (state_.input_type == InputType::TENSOR_BIN) - return executeTensorPipeline(); - - try { - if (state_.input_type != InputType::AUDIO_WAV) - throw PipelineError{"Unknown input type"}; - - // Load audio file - std::vector audio_data; - if (!loadAudioFile(state_.current_input_file, audio_data)) - throw PipelineError{"Failed to load input audio"}; - const size_t original_sample_count = audio_data.size(); - - // GCRN signal processing parameters (from model_config.h) - const size_t GCRN_HOP_SIZE = 160; // STFT_INPUT_SAMPLES - const size_t GCRN_MODEL_ELEMS = 322; // STFT_NUM_BINS*2 = 161*2 - const size_t GCRN_FFT_SIZE = 320; // actual FFT size (fft_size/2+1 = 161 bins) - const size_t GCRN_BATCH_N = 64; // max frames per STFT/ISTFT call - const size_t GCRN_PAD_FRAMES = 17; // padding frames to reach 401 per TVM window - // TVM window = 6×64 + 17 = 401 frames, fixed input shape [1,2,401,161] - const size_t GCRN_TOTAL_FRAMES = 6 * GCRN_BATCH_N + GCRN_PAD_FRAMES; // 401 - const size_t GCRN_NUM_BATCHES = 7; - - // Buffer sizes - size_t audio_batch_bytes = GCRN_BATCH_N * GCRN_HOP_SIZE * sizeof(int16_t); // 20480 bytes - size_t spectral_total_bytes = GCRN_TOTAL_FRAMES * GCRN_MODEL_ELEMS * sizeof(float); // 516488 bytes - - // Detect if pipeline has a TVM stage - const bool has_tvm_stage = std::any_of( - state_.pipeline_config.stages.begin(), state_.pipeline_config.stages.end(), - [](const PipelineStage& stage) { return stage.service == "tvm"; }); - - std::cout << "[App] GCRN configuration:" << std::endl; - std::cout << "[App] HOP=" << GCRN_HOP_SIZE << " MODEL_ELEMS=" << GCRN_MODEL_ELEMS - << " BATCH_N=" << GCRN_BATCH_N << " TOTAL_FRAMES=" << GCRN_TOTAL_FRAMES << std::endl; - std::cout << "[App] Spectral buffer: " << spectral_total_bytes << " bytes (401 frames)" << std::endl; - std::cout << "[App] TVM stage: " << (has_tvm_stage ? "enabled" : "bypassed") << std::endl; - - // Initialize generic client - if (!generic_client_->initialize()) - throw PipelineError{"Failed to initialize Generic Task client"}; - - DmaBuffer dma_buf1{audio_batch_bytes, "STFT input"}; - DmaBuffer dma_buf2{spectral_total_bytes, "STFT output"}; - DmaBuffer dma_buf3{spectral_total_bytes, "interleave output"}; - DmaBuffer dma_buf4{audio_batch_bytes, "ISTFT output"}; - DmaBuffer dma_buf5{spectral_total_bytes, "deinterleave output"}; - DmaBuffer dma_buf6{spectral_total_bytes, "interleave input"}; - - std::cout << "[App] DMA buffers:" << std::endl; - std::cout << "[App] buf1 (STFT audio in): phys=0x" << std::hex << dma_buf1->phys_addr - << std::dec << " size=" << dma_buf1->size << std::endl; - std::cout << "[App] buf2 (STFT out/deint in): phys=0x" << std::hex << dma_buf2->phys_addr - << std::dec << " size=" << dma_buf2->size << std::endl; - std::cout << "[App] buf3 (int out/ISTFT in): phys=0x" << std::hex << dma_buf3->phys_addr - << std::dec << " size=" << dma_buf3->size << std::endl; - std::cout << "[App] buf4 (ISTFT audio out): phys=0x" << std::hex << dma_buf4->phys_addr - << std::dec << " size=" << dma_buf4->size << std::endl; - std::cout << "[App] buf5 (deint out): phys=0x" << std::hex << dma_buf5->phys_addr - << std::dec << " size=" << dma_buf5->size << std::endl; - std::cout << "[App] buf6 (interleave in): phys=0x" << std::hex << dma_buf6->phys_addr - << std::dec << " size=" << dma_buf6->size << std::endl; - - // Find STFT and ISTFT stages once - const PipelineStage* stft_stage_ptr = nullptr; - const PipelineStage* istft_stage_ptr = nullptr; - for (const auto& stage : state_.pipeline_config.stages) { - if (stage.message_type == "C7X_MSG_STFT_ANALYZE") stft_stage_ptr = &stage; - else if (stage.message_type == "C7X_MSG_ISTFT_SYNTHESIZE") istft_stage_ptr = &stage; - } - if (!stft_stage_ptr || !istft_stage_ptr) - throw PipelineError{"Audio pipeline requires STFT and ISTFT stages"}; - - // Each full chunk consumes exactly 401 frames. - // Remaining frames < 401 are zero-padded to 401 and processed as a partial chunk; - // only the real frames are kept from its ISTFT output. - const size_t total_frames = - (audio_data.size() + GCRN_HOP_SIZE - 1) / GCRN_HOP_SIZE; - audio_data.resize(total_frames * GCRN_HOP_SIZE, int16_t{0}); - size_t num_full_chunks = total_frames / GCRN_TOTAL_FRAMES; - size_t partial_frames = total_frames % GCRN_TOTAL_FRAMES; - size_t num_chunks = num_full_chunks + (partial_frames > 0 ? 1 : 0); - - std::cout << "[App] Full file processing:" << std::endl; - std::cout << "[App] Total frames: " << total_frames - << " | Full chunks: " << num_full_chunks - << " | Partial chunk: " << partial_frames << " real frames" - << " (zero-padded to 401)" << std::endl; - - std::vector processed_audio_data; - processed_audio_data.reserve(total_frames * GCRN_HOP_SIZE); - - AudioStream audio_stream; - - uint64_t stft_out_base = dma_buf2->phys_addr; - uint64_t istft_src_base = dma_buf3->phys_addr; - - // Initialize TVM once before the chunk loop - if (has_tvm_stage && state_.tvm_artifacts_configured && !tvm_client_->is_initialized()) { - if (!tvm_client_->initialize(state_.tvm_artifacts_paths[0])) - throw PipelineError{"Failed to initialize TVM client"}; - tvm_client_->set_input_shape({1, 2, 401, 161}); - std::cout << "[App] TVM initialized, input shape [1,2,401,161]" << std::endl; - } - - auto t_total_start = std::chrono::steady_clock::now(); - - for (size_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) { - size_t chunk_frame_offset = chunk_idx * GCRN_TOTAL_FRAMES; - const size_t num_batches_this_chunk = GCRN_NUM_BATCHES; - // For the partial chunk, how many real frames it contains (0 means full chunk) - size_t real_frames_this_chunk = (chunk_idx == num_full_chunks && partial_frames > 0) - ? partial_frames : GCRN_TOTAL_FRAMES; - - auto t_chunk_start = std::chrono::steady_clock::now(); - - // ===================================================== - // Phase 1: STFT - // ===================================================== - auto t_stft_start = std::chrono::steady_clock::now(); - for (size_t batch_idx = 0; batch_idx < num_batches_this_chunk; batch_idx++) { - size_t frames_this_batch = (batch_idx < 6) ? GCRN_BATCH_N : GCRN_PAD_FRAMES; - size_t samples_this_batch = frames_this_batch * GCRN_HOP_SIZE; - size_t audio_offset = (chunk_frame_offset + batch_idx * GCRN_BATCH_N) * GCRN_HOP_SIZE; - size_t audio_bytes = samples_this_batch * sizeof(int16_t); - uint64_t spectral_offset = batch_idx * GCRN_BATCH_N * GCRN_MODEL_ELEMS * sizeof(float); - - dma_buf1.begin_cpu_access(); - std::fill_n(dma_buf1.data(), audio_batch_bytes, std::byte{}); - if (audio_offset < audio_data.size()) { - const size_t available = std::min( - samples_this_batch, audio_data.size() - audio_offset); - std::copy_n(audio_data.begin() + static_cast(audio_offset), - available, dma_buf1.data()); - } - dma_buf1.end_cpu_access(); - audio_stream.send_frame(0, dma_buf1.data(), audio_bytes); - - auto params = stft_stage_ptr->parameters; - params["input_buffer"] = hex_address(dma_buf1->phys_addr); - params["output_buffer"] = hex_address(stft_out_base + spectral_offset); - params["input_frame"] = std::to_string(frames_this_batch); - params["output_frame"] = std::to_string(frames_this_batch); - - if (debug_) - std::cout << "[App] STFT batch " << (batch_idx+1) << "/" << num_batches_this_chunk - << ": " << frames_this_batch << " frames" - << " in=0x" << std::hex << dma_buf1->phys_addr - << " out=0x" << (stft_out_base + spectral_offset) << std::dec << std::endl; - - auto r = generic_client_->process("C7X_MSG_STFT_ANALYZE", params); - if (!r.success) - throw PipelineError{"STFT batch " + std::to_string(batch_idx + 1) + - " failed: " + r.error_message}; - } - double t_stft_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t_stft_start).count() / 1000.0; - - // ===================================================== - // Phase 2: TVM - // ===================================================== - double t_tvm_ms = 0.0; - auto t_tvm_start = std::chrono::steady_clock::now(); - - if (debug_) - std::cout << "[App] Deinterleave: 0x" << std::hex << dma_buf2->phys_addr - << " -> 0x" << dma_buf5->phys_addr << std::dec << std::endl; - { - std::map params; - params["input_buffer"] = hex_address(dma_buf2->phys_addr); - params["output_buffer"] = hex_address(dma_buf5->phys_addr); - params["input_frame"] = std::to_string(GCRN_TOTAL_FRAMES); - params["fft_size"] = std::to_string(GCRN_FFT_SIZE); - params["flag"] = "0"; // deinterleave - auto r = generic_client_->process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); - if (!r.success) - throw PipelineError{"Deinterleave failed: " + r.error_message}; - } - deint_output_data_.resize(spectral_total_bytes / sizeof(float)); - inter_input_data_.resize(spectral_total_bytes / sizeof(float)); - - dma_buf5.begin_cpu_access(); - std::copy_n(dma_buf5.data(), deint_output_data_.size(), - deint_output_data_.begin()); - dma_buf5.end_cpu_access(); - // TVM only runs when pipeline has TVM stage - if (has_tvm_stage && tvm_client_->is_initialized()) { - if (!tvm_client_->run_inference(deint_output_data_, inter_input_data_)) - throw PipelineError{"TVM inference failed"}; - t_tvm_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t_tvm_start).count() / 1000.0; - } else { - inter_input_data_ = deint_output_data_; - } - if (inter_input_data_.size() != spectral_total_bytes / sizeof(float)) - throw PipelineError{"TVM output size does not match the speech pipeline"}; - - dma_buf6.begin_cpu_access(); - std::copy(inter_input_data_.begin(), inter_input_data_.end(), - dma_buf6.data()); - dma_buf6.end_cpu_access(); - if (debug_) - std::cout << "[App] Interleave: 0x" << std::hex << dma_buf6->phys_addr - << " -> 0x" << dma_buf3->phys_addr << std::dec << std::endl; - { - std::map params; - params["input_buffer"] = hex_address(dma_buf6->phys_addr); - params["output_buffer"] = hex_address(dma_buf3->phys_addr); - params["input_frame"] = std::to_string(GCRN_TOTAL_FRAMES); - params["fft_size"] = std::to_string(GCRN_FFT_SIZE); - params["flag"] = "1"; // interleave - auto r = generic_client_->process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); - if (!r.success) - throw PipelineError{"Interleave failed: " + r.error_message}; - } - - // ===================================================== - // Phase 3: ISTFT - // ===================================================== - auto t_istft_start = std::chrono::steady_clock::now(); - size_t real_samples_remaining = real_frames_this_chunk * GCRN_HOP_SIZE; - for (size_t batch_idx = 0; batch_idx < num_batches_this_chunk; batch_idx++) { - size_t frames_this_batch = (batch_idx < 6) ? GCRN_BATCH_N : GCRN_PAD_FRAMES; - size_t samples_this_batch = frames_this_batch * GCRN_HOP_SIZE; - size_t audio_offset = (chunk_frame_offset + batch_idx * GCRN_BATCH_N) * GCRN_HOP_SIZE; - uint64_t spectral_offset = batch_idx * GCRN_BATCH_N * GCRN_MODEL_ELEMS * sizeof(float); - auto params = istft_stage_ptr->parameters; - params["input_buffer"] = hex_address(istft_src_base + spectral_offset); - params["output_buffer"] = hex_address(dma_buf4->phys_addr); - params["input_frame"] = std::to_string(frames_this_batch); - params["output_frame"] = std::to_string(frames_this_batch); - - if (debug_) - std::cout << "[App] ISTFT batch " << (batch_idx+1) << "/" << num_batches_this_chunk - << ": " << frames_this_batch << " frames" - << " in=0x" << std::hex << (istft_src_base + spectral_offset) - << " out=0x" << dma_buf4->phys_addr << std::dec << std::endl; - - auto r = generic_client_->process("C7X_MSG_ISTFT_SYNTHESIZE", params); - if (!r.success) - throw PipelineError{"ISTFT batch " + std::to_string(batch_idx + 1) + - " failed: " + r.error_message}; - - dma_buf4.begin_cpu_access(); - const auto* out_ptr = dma_buf4.data(); - - if (debug_) { - std::cout << "[App] Frame | InRMS OutRMS | In[0..4] | Out[0..4]" << std::endl; - const size_t available_frames = std::min( - frames_this_batch, real_samples_remaining / GCRN_HOP_SIZE); - for (size_t f = 0; f < std::min(size_t{4}, available_frames); ++f) { - size_t in_off = audio_offset + f * GCRN_HOP_SIZE; - size_t out_off = f * GCRN_HOP_SIZE; - float in_sum = 0.0f, out_sum = 0.0f; - for (size_t i = 0; i < GCRN_HOP_SIZE; i++) { - const float s = static_cast(audio_data[in_off + i]) / 32768.0f; - const float o = static_cast(out_ptr[out_off + i]) / 32768.0f; - in_sum += s * s; - out_sum += o * o; - } - std::cout << "[App] " << std::setw(5) << (chunk_frame_offset + batch_idx*GCRN_BATCH_N + f + 1) - << " | " << std::fixed << std::setprecision(4) - << std::sqrt(in_sum / GCRN_HOP_SIZE) << " " - << std::sqrt(out_sum / GCRN_HOP_SIZE) - << " | In:"; - for (size_t i = 0; i < 5; i++) - std::cout << std::setw(6) << audio_data[in_off + i] << (i<4?",":""); - std::cout << " | Out:"; - for (size_t i = 0; i < 5; i++) - std::cout << std::setw(6) << out_ptr[out_off + i] << (i<4?",":""); - std::cout << std::endl; - } - } - - size_t samples_to_collect = std::min(samples_this_batch, real_samples_remaining); - std::copy_n(out_ptr, samples_to_collect, - std::back_inserter(processed_audio_data)); - real_samples_remaining -= samples_to_collect; - - dma_buf4.end_cpu_access(); - audio_stream.send_frame(1, out_ptr, samples_to_collect * sizeof(int16_t)); - } - double t_istft_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t_istft_start).count() / 1000.0; - - double t_chunk_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t_chunk_start).count() / 1000.0; - - std::cout << "[App] Chunk " << (chunk_idx+1) << "/" << num_chunks - << " [" << real_frames_this_chunk << " real frames" - << (real_frames_this_chunk < GCRN_TOTAL_FRAMES ? " + zero-pad" : "") << "]" - << " | STFT=" << std::fixed << std::setprecision(1) << t_stft_ms << "ms" - << " TVM=" << t_tvm_ms << "ms" - << " ISTFT=" << t_istft_ms << "ms" - << " total=" << t_chunk_ms << "ms" << std::endl; - } - - double t_total_ms = std::chrono::duration_cast( - std::chrono::steady_clock::now() - t_total_start).count() / 1000.0; - - std::cout << "[App] All chunks done | total=" << std::fixed << std::setprecision(1) - << t_total_ms << "ms | " << (processed_audio_data.size() / 160) << " output GCRN frames (" - << processed_audio_data.size() << " samples)" << std::endl; - - if (processed_audio_data.size() < original_sample_count) - throw PipelineError{"Speech pipeline produced fewer samples than expected"}; - processed_audio_data.resize(original_sample_count); - - std::string output_filename = "processed_output.wav"; - if (!saveAudioFile(output_filename, processed_audio_data)) - throw PipelineError{"Failed to save output file"}; - std::cout << "[App] Saved to " << output_filename << std::endl; - return CommandResult::SUCCESS; - } catch (const std::exception& error) { - std::cerr << "[App] Pipeline failed: " << error.what() << std::endl; - return CommandResult::ERROR; - } -} - - - -bool PipelineManager::loadAudioFile(const std::string& filename, std::vector& audio_data) -{ - SF_INFO sfinfo{}; - - std::unique_ptr infile{ - sf_open(filename.c_str(), SFM_READ, &sfinfo), &sf_close}; - if (!infile) { - std::cout << "[App] Error: Failed to open audio file: " << filename << std::endl; - return false; - } - - // Validate audio format - if (sfinfo.channels != 1) { - std::cout << "[App] Error: Audio must be mono (1 channel), got " << sfinfo.channels << " channels" << std::endl; - return false; - } - - if (sfinfo.frames <= 0) { - std::cout << "[App] Error: Audio file contains no samples" << std::endl; - return false; - } - - if (sfinfo.samplerate != 16000) { - std::cout << "[App] Error: Audio sample rate is " << sfinfo.samplerate - << "Hz; this pipeline requires 16kHz" << std::endl; - return false; - } - - std::cout << "[App] Audio file info: " << (sfinfo.frames / 160) << " GCRN frames (" - << sfinfo.frames << " samples), " - << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; - - // Read all audio data - audio_data.resize(sfinfo.frames); - const sf_count_t frames_read = sf_readf_short( - infile.get(), audio_data.data(), sfinfo.frames); - - if (frames_read < 0) { - std::cout << "[App] Error: Failed while reading audio data" << std::endl; - return false; - } - if (frames_read != sfinfo.frames) { - std::cout << "[App] Warning: Read " << frames_read << " frames, expected " << sfinfo.frames << std::endl; - audio_data.resize(frames_read); - } - - std::cout << "[App] Loaded " << audio_data.size() << " audio samples (" - << (static_cast(audio_data.size()) / sfinfo.samplerate) - << " seconds)" << std::endl; - - return true; -} - -bool PipelineManager::saveAudioFile(const std::string& filename, const std::vector& audio_data) -{ - if (audio_data.empty()) { - std::cout << "[App] Error: Refusing to write an empty audio file" << std::endl; - return false; - } - SF_INFO sfinfo{}; - - // Set output file parameters to match our audio format - sfinfo.samplerate = 16000; // 16kHz - sfinfo.channels = 1; // mono - sfinfo.format = SF_FORMAT_WAV | SF_FORMAT_PCM_16; // WAV file with 16-bit PCM - - std::unique_ptr outfile{ - sf_open(filename.c_str(), SFM_WRITE, &sfinfo), &sf_close}; - if (!outfile) { - std::cout << "[App] Error: Failed to create output audio file: " << filename << std::endl; - std::cout << "[App] Error details: " << sf_strerror(nullptr) << std::endl; - return false; - } - - std::cout << "[App] Saving audio to: " << filename << std::endl; - std::cout << "[App] Output file info: " << (audio_data.size() / 160) << " GCRN frames (" - << audio_data.size() << " samples), " - << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; - - // Write audio data to file - const sf_count_t frames_written = sf_writef_short( - outfile.get(), audio_data.data(), static_cast(audio_data.size())); - - if (frames_written != static_cast(audio_data.size())) { - std::cout << "[App] Warning: Wrote " << frames_written << " frames, expected " << audio_data.size() << std::endl; - return false; - } - - std::cout << "[App] Successfully saved " << (frames_written / 160) << " GCRN frames (" - << frames_written << " samples) to " << filename << std::endl; - std::cout << "[App] Duration: " - << (static_cast(frames_written) / sfinfo.samplerate) - << " seconds" << std::endl; - - return true; -} - -bool PipelineManager::saveTensorFile(const std::string& filename, const std::vector& tensor_data) { - if (tensor_data.empty()) { - std::cout << "[App] Error: Refusing to write an empty tensor" << std::endl; - return false; - } - std::ofstream file(filename, std::ios::binary); - if (!file.is_open()) { - std::cout << "[App] Error: Cannot open file for writing: " << filename << std::endl; - return false; - } - - file.write(reinterpret_cast(tensor_data.data()), - static_cast(tensor_data.size() * sizeof(float))); - if (!file) { - std::cout << "[App] Error: Failed while writing tensor data" << std::endl; - return false; - } - - std::cout << "[App] Successfully saved " << tensor_data.size() << " float values (" - << (tensor_data.size() * sizeof(float)) << " bytes) to " << filename << std::endl; - - return true; -} - -bool PipelineManager::loadBinTensor(const std::string& filename, std::vector& tensor_data) { - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if (!file.is_open()) { - std::cout << "[App] Error: Cannot open BIN file: " << filename << std::endl; - return false; - } - - // Get file size - const std::streamsize size = file.tellg(); - if (size <= 0 || size % static_cast(sizeof(float)) != 0) { - std::cout << "[App] Error: BIN file is not a non-empty float32 tensor" << std::endl; - return false; - } - file.seekg(0, std::ios::beg); - - // Calculate number of float values - const size_t num_floats = static_cast(size) / sizeof(float); - tensor_data.resize(num_floats); - - // Read binary data - if (!file.read(reinterpret_cast(tensor_data.data()), size)) { - std::cout << "[App] Error: Failed to read BIN file" << std::endl; return false; } diff --git a/example/edge-ai/src/stft_istft_pipeline.cpp b/example/edge-ai/src/stft_istft_pipeline.cpp new file mode 100644 index 0000000..545da49 --- /dev/null +++ b/example/edge-ai/src/stft_istft_pipeline.cpp @@ -0,0 +1,345 @@ +#include "stft_istft_pipeline.h" +#include "pipeline_common.h" +#include "audio_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +size_t require_param(const std::map& params, + const char* key, const char* stage) +{ + auto it = params.find(key); + if (it == params.end()) + throw PipelineError{std::string{"STFT stage missing required parameter: "} + key + + " (stage: " + stage + ")"}; + int v = std::stoi(it->second); + if (v <= 0) + throw PipelineError{std::string{"Parameter must be positive: "} + key}; + return static_cast(v); +} + +} // namespace + +PipelineManager::CommandResult run_stft_istft_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + bool debug) +{ + try { + if (state.input_type != PipelineManager::InputType::AUDIO_WAV) + throw PipelineError{"Unknown input type"}; + + // Find required stages + const PipelineManager::PipelineStage* stft_stage_ptr = nullptr; + const PipelineManager::PipelineStage* deint_stage_ptr = nullptr; + const PipelineManager::PipelineStage* inter_stage_ptr = nullptr; + const PipelineManager::PipelineStage* istft_stage_ptr = nullptr; + + for (const auto& stage : state.pipeline_config.stages) { + if (stage.message_type == "C7X_MSG_STFT_ANALYZE") stft_stage_ptr = &stage; + else if (stage.message_type == "C7X_MSG_ISTFT_SYNTHESIZE") istft_stage_ptr = &stage; + else if (stage.message_type == "C7X_DEINTERLEAVE_MSG_ANALYZE") { + auto it = stage.parameters.find("flag"); + if (it != stage.parameters.end() && it->second == "0") + deint_stage_ptr = &stage; + else + inter_stage_ptr = &stage; + } + } + + if (!stft_stage_ptr || !istft_stage_ptr || !deint_stage_ptr || !inter_stage_ptr) + throw PipelineError{"stft_istft pipeline requires STFT, deinterleave, interleave and ISTFT stages"}; + + // Read loop/buffer parameters from STFT stage + const auto& sp = stft_stage_ptr->parameters; + const size_t HOP_SIZE = require_param(sp, "hop_size", stft_stage_ptr->stage_id.c_str()); + const size_t MODEL_ELEMS = require_param(sp, "model_elems", stft_stage_ptr->stage_id.c_str()); + const size_t TOTAL_FRAMES = require_param(sp, "total_frames", stft_stage_ptr->stage_id.c_str()); + const size_t BATCH_N = require_param(sp, "batch_n", stft_stage_ptr->stage_id.c_str()); + const size_t PAD_FRAMES = TOTAL_FRAMES % BATCH_N; + const size_t NUM_BATCHES = (TOTAL_FRAMES + BATCH_N - 1) / BATCH_N; + + // Read parameters from ISTFT stage + const auto& ip = istft_stage_ptr->parameters; + const size_t ISTFT_HOP_SIZE = require_param(ip, "hop_size", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_MODEL_ELEMS = require_param(ip, "model_elems", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_TOTAL_FRAMES = require_param(ip, "total_frames", istft_stage_ptr->stage_id.c_str()); + const size_t ISTFT_BATCH_N = require_param(ip, "batch_n", istft_stage_ptr->stage_id.c_str()); + + // STFT-side buffer sizes (buf1, buf2, buf5) + const size_t audio_batch_bytes = BATCH_N * HOP_SIZE * sizeof(int16_t); + const size_t spectral_stft_bytes = TOTAL_FRAMES * MODEL_ELEMS * sizeof(float); + + // ISTFT-side buffer sizes (buf3, buf4, buf6) + const size_t audio_istft_batch_bytes = ISTFT_BATCH_N * ISTFT_HOP_SIZE * sizeof(int16_t); + const size_t spectral_istft_bytes = ISTFT_TOTAL_FRAMES * ISTFT_MODEL_ELEMS * sizeof(float); + + std::cout << "[App] STFT parameters: HOP=" << HOP_SIZE << " MODEL_ELEMS=" << MODEL_ELEMS + << " BATCH_N=" << BATCH_N << " TOTAL_FRAMES=" << TOTAL_FRAMES << std::endl; + std::cout << "[App] ISTFT parameters: HOP=" << ISTFT_HOP_SIZE << " MODEL_ELEMS=" << ISTFT_MODEL_ELEMS + << " BATCH_N=" << ISTFT_BATCH_N << " TOTAL_FRAMES=" << ISTFT_TOTAL_FRAMES << std::endl; + + // Load audio + std::vector audio_data; + if (!loadAudioFile(state.current_input_file, audio_data)) + throw PipelineError{"Failed to load input audio"}; + const size_t original_sample_count = audio_data.size(); + + if (!dsp_client.initialize(state.pipeline_config.dsp_config.proc_id, + state.pipeline_config.dsp_config.endpoint)) + throw PipelineError{"Failed to initialize DSP Task client"}; + + // Allocate DMA buffers + DmaBuffer dma_buf1{audio_batch_bytes, "STFT input"}; + DmaBuffer dma_buf2{spectral_stft_bytes, "STFT output"}; + DmaBuffer dma_buf3{spectral_istft_bytes, "interleave output"}; + DmaBuffer dma_buf4{audio_istft_batch_bytes, "ISTFT output"}; + DmaBuffer dma_buf5{spectral_stft_bytes, "deinterleave output"}; + DmaBuffer dma_buf6{spectral_istft_bytes, "interleave input"}; + + std::cout << "[App] DMA buffers:" << std::endl; + std::cout << "[App] buf1 (STFT audio in): phys=0x" << std::hex << dma_buf1->phys_addr + << std::dec << " size=" << dma_buf1->size << std::endl; + std::cout << "[App] buf2 (STFT out/deint in): phys=0x" << std::hex << dma_buf2->phys_addr + << std::dec << " size=" << dma_buf2->size << std::endl; + std::cout << "[App] buf3 (int out/ISTFT in): phys=0x" << std::hex << dma_buf3->phys_addr + << std::dec << " size=" << dma_buf3->size << std::endl; + std::cout << "[App] buf4 (ISTFT audio out): phys=0x" << std::hex << dma_buf4->phys_addr + << std::dec << " size=" << dma_buf4->size << std::endl; + std::cout << "[App] buf5 (deint out): phys=0x" << std::hex << dma_buf5->phys_addr + << std::dec << " size=" << dma_buf5->size << std::endl; + std::cout << "[App] buf6 (interleave in): phys=0x" << std::hex << dma_buf6->phys_addr + << std::dec << " size=" << dma_buf6->size << std::endl; + + // Chunk calculation + const size_t total_frames = + (audio_data.size() + HOP_SIZE - 1) / HOP_SIZE; + audio_data.resize(total_frames * HOP_SIZE, int16_t{0}); + const size_t num_full_chunks = total_frames / TOTAL_FRAMES; + const size_t partial_frames = total_frames % TOTAL_FRAMES; + const size_t num_chunks = num_full_chunks + (partial_frames > 0 ? 1 : 0); + + std::cout << "[App] Full file processing:" << std::endl; + std::cout << "[App] Total frames: " << total_frames + << " | Full chunks: " << num_full_chunks + << " | Partial chunk: " << partial_frames << " real frames" + << " (zero-padded to " << TOTAL_FRAMES << ")" << std::endl; + + std::vector processed_audio_data; + processed_audio_data.reserve(total_frames * HOP_SIZE); + + AudioStream audio_stream; + + uint64_t stft_out_base = dma_buf2->phys_addr; + uint64_t istft_src_base = dma_buf3->phys_addr; + + std::vector deint_output_data; + std::vector inter_input_data; + + auto t_total_start = std::chrono::steady_clock::now(); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) { + const size_t chunk_frame_offset = chunk_idx * TOTAL_FRAMES; + const size_t num_batches_chunk = NUM_BATCHES; + const size_t real_frames_chunk = (chunk_idx == num_full_chunks && partial_frames > 0) + ? partial_frames : TOTAL_FRAMES; + + auto t_chunk_start = std::chrono::steady_clock::now(); + + // Phase 1: STFT + auto t_stft_start = std::chrono::steady_clock::now(); + for (size_t batch_idx = 0; batch_idx < num_batches_chunk; batch_idx++) { + const size_t frames_this_batch = (batch_idx < NUM_BATCHES - 1) ? BATCH_N : PAD_FRAMES; + const size_t samples_this_batch = frames_this_batch * HOP_SIZE; + const size_t audio_offset = (chunk_frame_offset + batch_idx * BATCH_N) * HOP_SIZE; + const size_t audio_bytes = samples_this_batch * sizeof(int16_t); + const uint64_t spectral_offset = batch_idx * BATCH_N * MODEL_ELEMS * sizeof(float); + + dma_buf1.begin_cpu_access(); + std::fill_n(dma_buf1.data(), audio_batch_bytes, std::byte{}); + if (audio_offset < audio_data.size()) { + const size_t available = std::min(samples_this_batch, + audio_data.size() - audio_offset); + std::copy_n(audio_data.begin() + static_cast(audio_offset), + available, dma_buf1.data()); + } + dma_buf1.end_cpu_access(); + audio_stream.send_frame(0, dma_buf1.data(), audio_bytes); + + auto params = stft_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf1->phys_addr); + params["output_buffer"] = hex_address(stft_out_base + spectral_offset); + params["input_frame"] = std::to_string(frames_this_batch); + params["output_frame"] = std::to_string(frames_this_batch); + + if (debug) + std::cout << "[App] STFT batch " << (batch_idx+1) << "/" << num_batches_chunk + << ": " << frames_this_batch << " frames" + << " in=0x" << std::hex << dma_buf1->phys_addr + << " out=0x" << (stft_out_base + spectral_offset) << std::dec << std::endl; + + auto r = dsp_client.process("C7X_MSG_STFT_ANALYZE", params); + if (!r.success) + throw PipelineError{"STFT batch " + std::to_string(batch_idx + 1) + + " failed: " + r.error_message}; + } + double t_stft_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_stft_start).count() / 1000.0; + + // Phase 2: Deinterleave + if (debug) + std::cout << "[App] Deinterleave: 0x" << std::hex << dma_buf2->phys_addr + << " -> 0x" << dma_buf5->phys_addr << std::dec << std::endl; + { + auto params = deint_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf2->phys_addr); + params["output_buffer"] = hex_address(dma_buf5->phys_addr); + params["input_frame"] = std::to_string(TOTAL_FRAMES); + auto r = dsp_client.process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); + if (!r.success) + throw PipelineError{"Deinterleave failed: " + r.error_message}; + } + + // Phase 3: Pass-through (no TVM) + deint_output_data.resize(spectral_stft_bytes / sizeof(float)); + inter_input_data.resize(spectral_istft_bytes / sizeof(float)); + + dma_buf5.begin_cpu_access(); + std::copy_n(dma_buf5.data(), deint_output_data.size(), + deint_output_data.begin()); + dma_buf5.end_cpu_access(); + + inter_input_data = deint_output_data; + + // Phase 4: Interleave + dma_buf6.begin_cpu_access(); + std::copy(inter_input_data.begin(), inter_input_data.end(), + dma_buf6.data()); + dma_buf6.end_cpu_access(); + + if (debug) + std::cout << "[App] Interleave: 0x" << std::hex << dma_buf6->phys_addr + << " -> 0x" << dma_buf3->phys_addr << std::dec << std::endl; + { + auto params = inter_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_buf6->phys_addr); + params["output_buffer"] = hex_address(dma_buf3->phys_addr); + params["input_frame"] = std::to_string(TOTAL_FRAMES); + auto r = dsp_client.process("C7X_DEINTERLEAVE_MSG_ANALYZE", params); + if (!r.success) + throw PipelineError{"Interleave failed: " + r.error_message}; + } + + // Phase 5: ISTFT + const size_t ISTFT_PAD_FRAMES = ISTFT_TOTAL_FRAMES % ISTFT_BATCH_N; + const size_t ISTFT_NUM_BATCHES = (ISTFT_TOTAL_FRAMES + ISTFT_BATCH_N - 1) / ISTFT_BATCH_N; + auto t_istft_start = std::chrono::steady_clock::now(); + size_t real_samples_remaining = real_frames_chunk * ISTFT_HOP_SIZE; + for (size_t batch_idx = 0; batch_idx < ISTFT_NUM_BATCHES; batch_idx++) { + const size_t frames_this_batch = (batch_idx < ISTFT_NUM_BATCHES - 1) ? ISTFT_BATCH_N : ISTFT_PAD_FRAMES; + const size_t samples_this_batch = frames_this_batch * ISTFT_HOP_SIZE; + const size_t audio_offset = (chunk_frame_offset + batch_idx * ISTFT_BATCH_N) * ISTFT_HOP_SIZE; + const uint64_t spectral_offset = batch_idx * ISTFT_BATCH_N * ISTFT_MODEL_ELEMS * sizeof(float); + + auto params = istft_stage_ptr->parameters; + params["input_buffer"] = hex_address(istft_src_base + spectral_offset); + params["output_buffer"] = hex_address(dma_buf4->phys_addr); + params["input_frame"] = std::to_string(frames_this_batch); + params["output_frame"] = std::to_string(frames_this_batch); + + if (debug) + std::cout << "[App] ISTFT batch " << (batch_idx+1) << "/" << num_batches_chunk + << ": " << frames_this_batch << " frames" + << " in=0x" << std::hex << (istft_src_base + spectral_offset) + << " out=0x" << dma_buf4->phys_addr << std::dec << std::endl; + + auto r = dsp_client.process("C7X_MSG_ISTFT_SYNTHESIZE", params); + if (!r.success) + throw PipelineError{"ISTFT batch " + std::to_string(batch_idx + 1) + + " failed: " + r.error_message}; + + dma_buf4.begin_cpu_access(); + const auto* out_ptr = dma_buf4.data(); + + if (debug) { + std::cout << "[App] Frame | InRMS OutRMS | In[0..4] | Out[0..4]" << std::endl; + const size_t available_frames = std::min( + frames_this_batch, real_samples_remaining / ISTFT_HOP_SIZE); + for (size_t f = 0; f < std::min(size_t{4}, available_frames); ++f) { + const size_t in_off = audio_offset + f * ISTFT_HOP_SIZE; + const size_t out_off = f * ISTFT_HOP_SIZE; + float in_sum = 0.0f, out_sum = 0.0f; + for (size_t i = 0; i < ISTFT_HOP_SIZE; i++) { + const float s = static_cast(audio_data[in_off + i]) / 32768.0f; + const float o = static_cast(out_ptr[out_off + i]) / 32768.0f; + in_sum += s * s; + out_sum += o * o; + } + std::cout << "[App] " << std::setw(5) + << (chunk_frame_offset + batch_idx * ISTFT_BATCH_N + f + 1) + << " | " << std::fixed << std::setprecision(4) + << std::sqrt(in_sum / HOP_SIZE) << " " + << std::sqrt(out_sum / HOP_SIZE) + << " | In:"; + for (size_t i = 0; i < 5; i++) + std::cout << std::setw(6) << audio_data[in_off + i] << (i<4?",":""); + std::cout << " | Out:"; + for (size_t i = 0; i < 5; i++) + std::cout << std::setw(6) << out_ptr[out_off + i] << (i<4?",":""); + std::cout << std::endl; + } + } + + size_t samples_to_collect = std::min(samples_this_batch, real_samples_remaining); + std::copy_n(out_ptr, samples_to_collect, + std::back_inserter(processed_audio_data)); + real_samples_remaining -= samples_to_collect; + + dma_buf4.end_cpu_access(); + audio_stream.send_frame(1, out_ptr, samples_to_collect * sizeof(int16_t)); + } + double t_istft_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_istft_start).count() / 1000.0; + + double t_chunk_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_chunk_start).count() / 1000.0; + + std::cout << "[App] Chunk " << (chunk_idx+1) << "/" << num_chunks + << " [" << real_frames_chunk << " real frames" + << (real_frames_chunk < ISTFT_TOTAL_FRAMES ? " + zero-pad" : "") << "]" + << " | STFT=" << std::fixed << std::setprecision(1) << t_stft_ms << "ms" + << " TVM=0.0ms" + << " ISTFT=" << t_istft_ms << "ms" + << " total=" << t_chunk_ms << "ms" << std::endl; + } + + double t_total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_total_start).count() / 1000.0; + + std::cout << "[App] All chunks done | total=" << std::fixed << std::setprecision(1) + << t_total_ms << "ms | " << (processed_audio_data.size() / ISTFT_HOP_SIZE) + << " output frames (" << processed_audio_data.size() << " samples)" << std::endl; + + if (processed_audio_data.size() < original_sample_count) + throw PipelineError{"Pipeline produced fewer samples than expected"}; + processed_audio_data.resize(original_sample_count); + + std::string output_filename = "processed_output.wav"; + if (!saveAudioFile(output_filename, processed_audio_data)) + throw PipelineError{"Failed to save output file"}; + std::cout << "[App] Saved to " << output_filename << std::endl; + return PipelineManager::CommandResult::SUCCESS; + } catch (const std::exception& error) { + std::cerr << "[App] Pipeline failed: " << error.what() << std::endl; + return PipelineManager::CommandResult::ERROR; + } +} diff --git a/example/edge-ai/src/tvm_pipeline.cpp b/example/edge-ai/src/tvm_pipeline.cpp new file mode 100644 index 0000000..9d2f0df --- /dev/null +++ b/example/edge-ai/src/tvm_pipeline.cpp @@ -0,0 +1,102 @@ +#include "tvm_pipeline.h" +#include +#include +#include +#include + +namespace { + +bool saveTensorFile(const std::string& filename, const std::vector& tensor_data) +{ + if (tensor_data.empty()) { + std::cout << "[App] Error: Refusing to write an empty tensor" << std::endl; + return false; + } + std::ofstream file(filename, std::ios::binary); + if (!file.is_open()) { + std::cout << "[App] Error: Cannot open file for writing: " << filename << std::endl; + return false; + } + + file.write(reinterpret_cast(tensor_data.data()), + static_cast(tensor_data.size() * sizeof(float))); + if (!file) { + std::cout << "[App] Error: Failed while writing tensor data" << std::endl; + return false; + } + + std::cout << "[App] Successfully saved " << tensor_data.size() << " float values (" + << (tensor_data.size() * sizeof(float)) << " bytes) to " << filename << std::endl; + + return true; +} + +bool loadBinTensor(const std::string& filename, std::vector& tensor_data) +{ + std::ifstream file(filename, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + std::cout << "[App] Error: Cannot open BIN file: " << filename << std::endl; + return false; + } + + const std::streamsize size = file.tellg(); + if (size <= 0 || size % static_cast(sizeof(float)) != 0) { + std::cout << "[App] Error: BIN file is not a non-empty float32 tensor" << std::endl; + return false; + } + file.seekg(0, std::ios::beg); + + const size_t num_floats = static_cast(size) / sizeof(float); + tensor_data.resize(num_floats); + + if (!file.read(reinterpret_cast(tensor_data.data()), size)) { + std::cout << "[App] Error: Failed to read BIN file" << std::endl; + return false; + } + + return true; +} + +} // namespace + +PipelineManager::CommandResult run_tvm_pipeline( + PipelineManager::State& state, + TvmInferenceClient& tvm_client) +{ + std::cout << "\n[App] === Executing Tensor-Only Pipeline ===" << std::endl; + + if (state.pipeline_config.stages.size() != 1 || + state.pipeline_config.stages[0].service != "tvm") { + std::cout << "[App] Error: Tensor pipeline must have exactly 1 TVM stage" << std::endl; + return PipelineManager::CommandResult::ERROR; + } + + std::cout << "[App] Running TVM inference" << std::endl; + std::cout << "[App] Artifacts: " << state.tvm_artifacts_paths[0] << std::endl; + std::cout << "[App] Input: " << state.current_input_file << std::endl; + + if (!tvm_client.initialize(state.tvm_artifacts_paths[0])) { + std::cout << "[App] Error: Failed to initialize TVM client" << std::endl; + return PipelineManager::CommandResult::ERROR; + } + + if (!tvm_client.run_inference(state.current_input_file)) { + std::cout << "[App] Error: TVM inference failed" << std::endl; + return PipelineManager::CommandResult::ERROR; + } + + const std::filesystem::path input_path{state.current_input_file}; + const std::string output_file = + (input_path.parent_path() / (input_path.stem().string() + "_output.bin")).string(); + + const std::vector& output = tvm_client.get_output(); + if (!saveTensorFile(output_file, output)) { + std::cout << "[App] Error: Failed to save output tensor" << std::endl; + return PipelineManager::CommandResult::ERROR; + } + + std::cout << "[App] Output saved to: " << output_file << std::endl; + std::cout << "[App] Pipeline completed successfully" << std::endl; + + return PipelineManager::CommandResult::SUCCESS; +} From 12813b8394de60fe98b78a224eec5bf6899684eb Mon Sep 17 00:00:00 2001 From: Paresh Bhagat Date: Fri, 21 Aug 2026 10:43:08 +0530 Subject: [PATCH 2/7] example: edge-ai: Add overlapping to audio enhancement pipeline Replace non-overlapping chunk processing with overlap-save approach: - OVERLAP_FRAMES=100, HOP_FRAMES=301, T_FRAMES=50 - Consecutive chunks overlap by 100 frames (1600 samples at 160 hop) - trim_reconstruct: first chunk keeps [0..end-T], middle chunks keep [T..end-T], last chunk keeps [T..real_end] Signed-off-by: Paresh Bhagat --- .../src/audio_enhancement_pipeline.cpp | 101 +++++++++++------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/example/edge-ai/src/audio_enhancement_pipeline.cpp b/example/edge-ai/src/audio_enhancement_pipeline.cpp index 444c73e..7550673 100644 --- a/example/edge-ai/src/audio_enhancement_pipeline.cpp +++ b/example/edge-ai/src/audio_enhancement_pipeline.cpp @@ -128,19 +128,31 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( std::cout << "[App] buf6 (interleave in): phys=0x" << std::hex << dma_buf6->phys_addr << std::dec << " size=" << dma_buf6->size << std::endl; - // Chunk calculation - const size_t total_frames = - (audio_data.size() + HOP_SIZE - 1) / HOP_SIZE; - audio_data.resize(total_frames * HOP_SIZE, int16_t{0}); - const size_t num_full_chunks = total_frames / TOTAL_FRAMES; - const size_t partial_frames = total_frames % TOTAL_FRAMES; - const size_t num_chunks = num_full_chunks + (partial_frames > 0 ? 1 : 0); - - std::cout << "[App] Full file processing:" << std::endl; - std::cout << "[App] Total frames: " << total_frames - << " | Full chunks: " << num_full_chunks - << " | Partial chunk: " << partial_frames << " real frames" - << " (zero-padded to " << TOTAL_FRAMES << ")" << std::endl; + // Overlap-save chunking parameters (sample level) + const size_t OVERLAP_FRAMES = 100; // frames of overlap between chunks + const size_t T_FRAMES = OVERLAP_FRAMES / 2; // trim from each edge + const size_t HOP_FRAMES = TOTAL_FRAMES - OVERLAP_FRAMES; + const size_t T_SAMPLES = T_FRAMES * HOP_SIZE; + const size_t HOP_SAMPLES = HOP_FRAMES * HOP_SIZE; + const size_t CHUNK_SAMPLES = TOTAL_FRAMES * HOP_SIZE; + + // Pad audio so last chunk is full TOTAL_FRAMES + const size_t n_samples = audio_data.size(); + size_t n_chunks; + if (n_samples <= CHUNK_SAMPLES) { + n_chunks = 1; + } else { + n_chunks = 1 + static_cast( + std::ceil(static_cast(n_samples - CHUNK_SAMPLES) / HOP_SAMPLES)); + } + const size_t padded_len = (n_chunks - 1) * HOP_SAMPLES + CHUNK_SAMPLES; + audio_data.resize(padded_len, int16_t{0}); + + std::cout << "[App] Overlap-save chunking:" << std::endl; + std::cout << "[App] TOTAL_FRAMES=" << TOTAL_FRAMES + << " OVERLAP_FRAMES=" << OVERLAP_FRAMES + << " HOP_FRAMES=" << HOP_FRAMES + << " n_chunks=" << n_chunks << std::endl; // Initialize TVM — read input shape from TVM stage parameters if (tvm_stage_ptr && state.tvm_artifacts_configured && !tvm_client.is_initialized()) { @@ -169,7 +181,7 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( } std::vector processed_audio_data; - processed_audio_data.reserve(total_frames * HOP_SIZE); + processed_audio_data.reserve(n_samples); AudioStream audio_stream; @@ -181,11 +193,15 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( auto t_total_start = std::chrono::steady_clock::now(); - for (size_t chunk_idx = 0; chunk_idx < num_chunks; chunk_idx++) { - const size_t chunk_frame_offset = chunk_idx * TOTAL_FRAMES; + for (size_t chunk_idx = 0; chunk_idx < n_chunks; chunk_idx++) { + // Sample offset for this chunk (overlap-save: chunks are HOP_SAMPLES apart) + const size_t chunk_sample_offset = chunk_idx * HOP_SAMPLES; + const size_t chunk_frame_offset = chunk_sample_offset / HOP_SIZE; const size_t num_batches_chunk = NUM_BATCHES; - const size_t real_frames_chunk = (chunk_idx == num_full_chunks && partial_frames > 0) - ? partial_frames : TOTAL_FRAMES; + + // Trim-reconstruct: which output samples to keep from this chunk + const size_t lo_sample = (chunk_idx == 0) ? 0 : T_SAMPLES; + const size_t hi_sample = (chunk_idx == n_chunks - 1) ? CHUNK_SAMPLES : CHUNK_SAMPLES - T_SAMPLES; auto t_chunk_start = std::chrono::steady_clock::now(); @@ -194,7 +210,7 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( for (size_t batch_idx = 0; batch_idx < num_batches_chunk; batch_idx++) { const size_t frames_this_batch = (batch_idx < NUM_BATCHES - 1) ? BATCH_N : PAD_FRAMES; const size_t samples_this_batch = frames_this_batch * HOP_SIZE; - const size_t audio_offset = (chunk_frame_offset + batch_idx * BATCH_N) * HOP_SIZE; + const size_t audio_offset = chunk_sample_offset + batch_idx * BATCH_N * HOP_SIZE; const size_t audio_bytes = samples_this_batch * sizeof(int16_t); const uint64_t spectral_offset = batch_idx * BATCH_N * MODEL_ELEMS * sizeof(float); @@ -286,15 +302,17 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( throw PipelineError{"Interleave failed: " + r.error_message}; } - // Phase 5: ISTFT + // Phase 5: ISTFT — collect full chunk output into temporary buffer const size_t ISTFT_PAD_FRAMES = ISTFT_TOTAL_FRAMES % ISTFT_BATCH_N; const size_t ISTFT_NUM_BATCHES = (ISTFT_TOTAL_FRAMES + ISTFT_BATCH_N - 1) / ISTFT_BATCH_N; + std::vector chunk_output; + chunk_output.reserve(CHUNK_SAMPLES); + auto t_istft_start = std::chrono::steady_clock::now(); - size_t real_samples_remaining = real_frames_chunk * ISTFT_HOP_SIZE; for (size_t batch_idx = 0; batch_idx < ISTFT_NUM_BATCHES; batch_idx++) { const size_t frames_this_batch = (batch_idx < ISTFT_NUM_BATCHES - 1) ? ISTFT_BATCH_N : ISTFT_PAD_FRAMES; const size_t samples_this_batch = frames_this_batch * ISTFT_HOP_SIZE; - const size_t audio_offset = (chunk_frame_offset + batch_idx * ISTFT_BATCH_N) * ISTFT_HOP_SIZE; + const size_t audio_offset = chunk_sample_offset + batch_idx * ISTFT_BATCH_N * ISTFT_HOP_SIZE; const uint64_t spectral_offset = batch_idx * ISTFT_BATCH_N * ISTFT_MODEL_ELEMS * sizeof(float); auto params = istft_stage_ptr->parameters; @@ -304,7 +322,7 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( params["output_frame"] = std::to_string(frames_this_batch); if (debug) - std::cout << "[App] ISTFT batch " << (batch_idx+1) << "/" << num_batches_chunk + std::cout << "[App] ISTFT batch " << (batch_idx+1) << "/" << ISTFT_NUM_BATCHES << ": " << frames_this_batch << " frames" << " in=0x" << std::hex << (istft_src_base + spectral_offset) << " out=0x" << dma_buf4->phys_addr << std::dec << std::endl; @@ -319,9 +337,7 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( if (debug) { std::cout << "[App] Frame | InRMS OutRMS | In[0..4] | Out[0..4]" << std::endl; - const size_t available_frames = std::min( - frames_this_batch, real_samples_remaining / ISTFT_HOP_SIZE); - for (size_t f = 0; f < std::min(size_t{4}, available_frames); ++f) { + for (size_t f = 0; f < std::min(size_t{4}, frames_this_batch); ++f) { const size_t in_off = audio_offset + f * ISTFT_HOP_SIZE; const size_t out_off = f * ISTFT_HOP_SIZE; float in_sum = 0.0f, out_sum = 0.0f; @@ -334,8 +350,8 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( std::cout << "[App] " << std::setw(5) << (chunk_frame_offset + batch_idx * ISTFT_BATCH_N + f + 1) << " | " << std::fixed << std::setprecision(4) - << std::sqrt(in_sum / HOP_SIZE) << " " - << std::sqrt(out_sum / HOP_SIZE) + << std::sqrt(in_sum / ISTFT_HOP_SIZE) << " " + << std::sqrt(out_sum / ISTFT_HOP_SIZE) << " | In:"; for (size_t i = 0; i < 5; i++) std::cout << std::setw(6) << audio_data[in_off + i] << (i<4?",":""); @@ -346,23 +362,28 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( } } - size_t samples_to_collect = std::min(samples_this_batch, real_samples_remaining); - std::copy_n(out_ptr, samples_to_collect, - std::back_inserter(processed_audio_data)); - real_samples_remaining -= samples_to_collect; - + std::copy_n(out_ptr, samples_this_batch, std::back_inserter(chunk_output)); dma_buf4.end_cpu_access(); - audio_stream.send_frame(1, out_ptr, samples_to_collect * sizeof(int16_t)); } double t_istft_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - t_istft_start).count() / 1000.0; + // Trim-reconstruct: keep [lo_sample..hi_sample) from this chunk + const size_t keep_end = std::min(hi_sample, chunk_output.size()); + if (lo_sample < keep_end) { + std::copy(chunk_output.begin() + static_cast(lo_sample), + chunk_output.begin() + static_cast(keep_end), + std::back_inserter(processed_audio_data)); + audio_stream.send_frame(1, + chunk_output.data() + lo_sample, + (keep_end - lo_sample) * sizeof(int16_t)); + } + double t_chunk_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - t_chunk_start).count() / 1000.0; - std::cout << "[App] Chunk " << (chunk_idx+1) << "/" << num_chunks - << " [" << real_frames_chunk << " real frames" - << (real_frames_chunk < ISTFT_TOTAL_FRAMES ? " + zero-pad" : "") << "]" + std::cout << "[App] Chunk " << (chunk_idx+1) << "/" << n_chunks + << " [keep samples " << lo_sample << ".." << keep_end << "]" << " | STFT=" << std::fixed << std::setprecision(1) << t_stft_ms << "ms" << " TVM=" << t_tvm_ms << "ms" << " ISTFT=" << t_istft_ms << "ms" @@ -376,9 +397,9 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( << t_total_ms << "ms | " << (processed_audio_data.size() / ISTFT_HOP_SIZE) << " output frames (" << processed_audio_data.size() << " samples)" << std::endl; - if (processed_audio_data.size() < original_sample_count) - throw PipelineError{"Pipeline produced fewer samples than expected"}; - processed_audio_data.resize(original_sample_count); + // Trim to original length (remove any zero-padding) + if (processed_audio_data.size() > original_sample_count) + processed_audio_data.resize(original_sample_count); std::string output_filename = "processed_output.wav"; if (!saveAudioFile(output_filename, processed_audio_data)) From a25fd302c27062fe7cb59f422a47fcb7255ec2e3 Mon Sep 17 00:00:00 2001 From: Vishnu Singh Date: Fri, 21 Aug 2026 11:38:45 +0530 Subject: [PATCH 3/7] Add demo-manager daemon and TVM model preload service Introduces a Unix domain socket daemon (demo-manager) that manages the lifecycle of DSP demos on the board. The daemon accepts JSON commands (list/run/stop/status/preload/quit) over /var/run/demo-manager.sock and handles two demo classes: - edge-ai: automatically preloads the TVM model before launch to eliminate cold-start latency from Module::LoadFromFile - dsp-compute (2dfft, audio-offload, sigchain-biquad): invalidates the TVM model cache after completion, since these demos load their own DSP firmware and overwrite TVM runtime state Also adds: - demo-ctl: CLI client for sending commands to the daemon - tvm_model_daemon: persistent service that keeps the TVM model loaded and serves inference requests over a socket, enabling the fast-path in TvmInferenceClient - systemd service files for demo-manager and tvm-model-daemon - TvmInferenceClient: fast-path to delegate to daemon, avoiding per-run model reload; synchronize_dma_buffer now takes an explicit fd Signed-off-by: Vishnu Singh Signed-off-by: Paresh Bhagat --- demo_manager/CMakeLists.txt | 21 + demo_manager/demo-manager.service | 15 + demo_manager/src/demo_ctl.cpp | 122 +++++ demo_manager/src/demo_manager_daemon.cpp | 417 ++++++++++++++++++ example/edge-ai/CMakeLists.txt | 52 ++- example/edge-ai/include/pipeline_manager.h | 10 + example/edge-ai/include/tvm_daemon_proto.h | 37 ++ .../edge-ai/include/tvm_inference_client.h | 25 +- example/edge-ai/src/main.cpp | 12 +- example/edge-ai/src/pipeline_manager.cpp | 57 +++ example/edge-ai/src/tvm_inference_client.cpp | 212 ++++++++- example/edge-ai/src/tvm_model_daemon.cpp | 182 ++++++++ example/edge-ai/tvm-model-daemon.service | 17 + example/edge-ai/tvm-model-preload.service | 17 + 14 files changed, 1185 insertions(+), 11 deletions(-) create mode 100644 demo_manager/CMakeLists.txt create mode 100644 demo_manager/demo-manager.service create mode 100644 demo_manager/src/demo_ctl.cpp create mode 100644 demo_manager/src/demo_manager_daemon.cpp create mode 100644 example/edge-ai/include/tvm_daemon_proto.h create mode 100644 example/edge-ai/src/tvm_model_daemon.cpp create mode 100644 example/edge-ai/tvm-model-daemon.service create mode 100644 example/edge-ai/tvm-model-preload.service diff --git a/demo_manager/CMakeLists.txt b/demo_manager/CMakeLists.txt new file mode 100644 index 0000000..2623f54 --- /dev/null +++ b/demo_manager/CMakeLists.txt @@ -0,0 +1,21 @@ +enable_language(CXX) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(JSON_C REQUIRED json-c) + +# demo-manager: top-level daemon that manages all demo applications +add_executable(demo-manager src/demo_manager_daemon.cpp) + +target_compile_options(demo-manager PRIVATE -Wall -Wextra -O2) +target_include_directories(demo-manager PRIVATE ${JSON_C_INCLUDE_DIRS}) +target_link_libraries(demo-manager PRIVATE ${JSON_C_LIBRARIES}) + +# demo-ctl: CLI client to send commands to the daemon +add_executable(demo-ctl src/demo_ctl.cpp) + +target_compile_options(demo-ctl PRIVATE -Wall -Wextra -O2) +target_include_directories(demo-ctl PRIVATE ${JSON_C_INCLUDE_DIRS}) +target_link_libraries(demo-ctl PRIVATE ${JSON_C_LIBRARIES}) + +install(TARGETS demo-manager demo-ctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(FILES demo-manager.service DESTINATION ${SYSTEMD_SYSTEM_UNITDIR}) diff --git a/demo_manager/demo-manager.service b/demo_manager/demo-manager.service new file mode 100644 index 0000000..c67ea89 --- /dev/null +++ b/demo_manager/demo-manager.service @@ -0,0 +1,15 @@ +[Unit] +Description=Demo Manager Daemon +Documentation=man:demo-manager(1) +After=sysinit.target local-fs.target + +[Service] +Type=simple +ExecStart=/usr/bin/demo-manager +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/demo_manager/src/demo_ctl.cpp b/demo_manager/src/demo_ctl.cpp new file mode 100644 index 0000000..1f4db96 --- /dev/null +++ b/demo_manager/src/demo_ctl.cpp @@ -0,0 +1,122 @@ +// demo-ctl - CLI client for the demo-manager daemon +// +// Usage: +// demo-ctl list +// demo-ctl status +// demo-ctl run [extra args...] +// demo-ctl stop +// demo-ctl preload +// demo-ctl quit + +#include +#include +#include +#include + +#include +#include +#include + +#include + +static constexpr const char *SOCKET_PATH = "/var/run/demo-manager.sock"; + +static void usage(const char *prog) { + fprintf(stderr, + "Usage: %s [args]\n\n" + "Commands:\n" + " list List available demos\n" + " status Show currently running demo\n" + " run [args...] Start a demo (args forwarded to executable)\n" + " stop Stop the running demo\n" + " preload Preload the TVM model for edge-ai\n" + " quit Shut down the demo-manager daemon\n\n" + "Examples:\n" + " %s run edge-ai\n" + " %s run edge-ai pipeline_audio_enhancement.json\n" + " %s run edge-ai pipeline_tvm_inference.json --debug\n" + " %s run 2dfft\n" + " %s stop\n", + prog, prog, prog, prog, prog, prog); +} + +static int send_command(const std::string &json_str) { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { perror("socket"); return 1; } + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_PATH, sizeof(addr.sun_path) - 1); + + if (connect(fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { + fprintf(stderr, "Cannot connect to demo-manager (%s): %s\n" + "Is the daemon running? Try: systemctl start demo-manager\n", + SOCKET_PATH, strerror(errno)); + close(fd); + return 1; + } + + write(fd, json_str.c_str(), json_str.size()); + + // Accumulate response (daemon sends one JSON line) + std::string buf; + char chunk[1024]; + ssize_t n; + while ((n = read(fd, chunk, sizeof(chunk))) > 0) + buf.append(chunk, static_cast(n)); + close(fd); + + if (buf.empty()) { fprintf(stderr, "no response from daemon\n"); return 1; } + + // Pretty-print and check status + json_object *resp = json_tokener_parse(buf.c_str()); + if (!resp) { printf("%s\n", buf.c_str()); return 0; } + + printf("%s\n", json_object_to_json_string_ext(resp, + JSON_C_TO_STRING_PRETTY | JSON_C_TO_STRING_SPACED)); + + int rc = 0; + json_object *status_obj = nullptr; + if (json_object_object_get_ex(resp, "status", &status_obj) && + strcmp(json_object_get_string(status_obj), "error") == 0) + rc = 1; + + json_object_put(resp); + return rc; +} + +int main(int argc, char **argv) { + if (argc < 2) { usage(argv[0]); return 1; } + + std::string cmd = argv[1]; + json_object *req = json_object_new_object(); + + if (cmd == "list" || cmd == "status" || cmd == "stop" || + cmd == "preload" || cmd == "quit") { + json_object_object_add(req, "cmd", json_object_new_string(cmd.c_str())); + + } else if (cmd == "run") { + if (argc < 3) { + fprintf(stderr, "'run' requires a demo name\nAvailable: edge-ai, 2dfft, audio-offload, sigchain-biquad\n"); + json_object_put(req); + return 1; + } + json_object_object_add(req, "cmd", json_object_new_string("run")); + json_object_object_add(req, "demo", json_object_new_string(argv[2])); + if (argc > 3) { + json_object *arr = json_object_new_array(); + for (int i = 3; i < argc; i++) + json_object_array_add(arr, json_object_new_string(argv[i])); + json_object_object_add(req, "args", arr); + } + } else { + fprintf(stderr, "Unknown command: %s\n\n", cmd.c_str()); + usage(argv[0]); + json_object_put(req); + return 1; + } + + std::string json_str = json_object_to_json_string(req); + json_object_put(req); + return send_command(json_str); +} diff --git a/demo_manager/src/demo_manager_daemon.cpp b/demo_manager/src/demo_manager_daemon.cpp new file mode 100644 index 0000000..985d2fa --- /dev/null +++ b/demo_manager/src/demo_manager_daemon.cpp @@ -0,0 +1,417 @@ +// Demo Manager Daemon +// Listens on a Unix domain socket for JSON commands to run/stop demos. +// Automatically preloads the TVM model before edge-ai demos and invalidates +// the TVM model cache after any DSP compute demo completes or is stopped +// (since DSP compute demos load their own firmware, overwriting the TVM state). +// +// Socket: /var/run/demo-manager.sock +// Protocol: newline-terminated JSON request/response +// +// Commands: +// {"cmd":"list"} - list available demos +// {"cmd":"status"} - show running demo +// {"cmd":"run","demo":""} - start a demo +// {"cmd":"run","demo":"","args":[...]} - start with extra args +// {"cmd":"stop"} - stop running demo +// {"cmd":"preload"} - preload TVM model +// {"cmd":"quit"} - shutdown daemon + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static constexpr const char *SOCKET_PATH = "/var/run/demo-manager.sock"; +static constexpr const char *PID_FILE = "/var/run/demo-manager.pid"; +static constexpr const char *TVM_CACHE = "/var/lib/tvm_inference/loaded_model"; +static constexpr const char *PRELOAD_EXE = "/usr/bin/rpmsg_inference_example"; + +struct DemoEntry { + const char *name; + const char *description; + const char *executable; + bool is_edge_ai; // auto-preload TVM before run + bool is_dsp_compute; // invalidate TVM cache after run +}; + +static constexpr DemoEntry DEMOS[] = { + { + "edge-ai", + "TVM ML inference (GCRN speech enhancement) with C7x DSP offload", + "/usr/bin/rpmsg_inference_example", + true, false + }, + { + "2dfft", + "2D FFT computation offloaded to C7x DSP", + "/usr/bin/rpmsg_2dfft_example", + false, true + }, + { + "audio-offload", + "FFT-based audio processing with C7x DSP offload", + "/usr/bin/rpmsg_audio_offload_example", + false, true + }, + { + "sigchain-biquad", + "3-stage parametric equalizer biquad cascade on C7x DSP", + "/usr/bin/rpmsg_sigchain_biquad_example", + false, true + }, +}; +static constexpr int N_DEMOS = static_cast(sizeof(DEMOS) / sizeof(DEMOS[0])); + +// Global state - accessed from signal handlers so volatile sig_atomic_t +static volatile sig_atomic_t g_quit = 0; +static volatile sig_atomic_t g_child_done = 0; +static pid_t g_child_pid = -1; +static bool g_invalidate_on_exit = false; +static char g_active_demo[64] = {}; + +static void on_signal(int sig) { + if (sig == SIGTERM || sig == SIGINT) g_quit = 1; +} +static void on_sigchld(int) { g_child_done = 1; } + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +static bool file_exists(const char *path) { return access(path, F_OK) == 0; } + +static void invalidate_tvm_cache() { + if (!file_exists(TVM_CACHE)) return; + if (unlink(TVM_CACHE) == 0) + syslog(LOG_INFO, "TVM model cache invalidated (%s)", TVM_CACHE); + else + syslog(LOG_WARNING, "Failed to remove TVM cache %s: %s", TVM_CACHE, strerror(errno)); +} + +// Build a null-terminated argv array from a vector and exec, returning exit code. +static int run_sync(const std::vector &args) { + std::vector av; + av.reserve(args.size() + 1); + for (auto &s : args) av.push_back(s.c_str()); + av.push_back(nullptr); + + pid_t pid = fork(); + if (pid < 0) { + syslog(LOG_ERR, "fork: %s", strerror(errno)); + return -1; + } + if (pid == 0) { + execv(av[0], const_cast(av.data())); + _exit(127); + } + int status = 0; + waitpid(pid, &status, 0); + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +// Fork and exec without waiting - returns child pid, or -1 on error. +static pid_t launch_async(const std::vector &args) { + std::vector av; + av.reserve(args.size() + 1); + for (auto &s : args) av.push_back(s.c_str()); + av.push_back(nullptr); + + pid_t pid = fork(); + if (pid < 0) { syslog(LOG_ERR, "fork: %s", strerror(errno)); return -1; } + if (pid == 0) { + execv(av[0], const_cast(av.data())); + _exit(127); + } + return pid; +} + +// Called from main loop whenever g_child_done is set. +static void reap_child() { + g_child_done = 0; + if (g_child_pid < 0) return; + + int status = 0; + pid_t r = waitpid(g_child_pid, &status, WNOHANG); + if (r != g_child_pid) return; + + int code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; + syslog(LOG_INFO, "Demo '%s' pid=%d exited with code %d", + g_active_demo, (int)r, code); + + if (g_invalidate_on_exit) invalidate_tvm_cache(); + + g_child_pid = -1; + g_active_demo[0] = '\0'; + g_invalidate_on_exit = false; +} + +// Stop the active child cleanly (SIGTERM, then SIGKILL after 5s). +static void stop_child() { + if (g_child_pid < 0) return; + + syslog(LOG_INFO, "Stopping demo '%s' pid=%d", g_active_demo, (int)g_child_pid); + kill(g_child_pid, SIGTERM); + + for (int i = 0; i < 50; i++) { + usleep(100000); // 100 ms per tick + int status = 0; + if (waitpid(g_child_pid, &status, WNOHANG) == g_child_pid) goto done; + } + kill(g_child_pid, SIGKILL); + waitpid(g_child_pid, nullptr, 0); + +done: + if (g_invalidate_on_exit) invalidate_tvm_cache(); + g_child_pid = -1; + g_active_demo[0] = '\0'; + g_invalidate_on_exit = false; +} + +// --------------------------------------------------------------------------- +// JSON response builders +// --------------------------------------------------------------------------- + +static std::string resp_ok(json_object *data = nullptr) { + json_object *o = json_object_new_object(); + json_object_object_add(o, "status", json_object_new_string("ok")); + if (data) json_object_object_add(o, "data", data); + std::string s = json_object_to_json_string(o); + json_object_put(o); + return s + "\n"; +} + +static std::string resp_err(const std::string &msg) { + json_object *o = json_object_new_object(); + json_object_object_add(o, "status", json_object_new_string("error")); + json_object_object_add(o, "message", json_object_new_string(msg.c_str())); + std::string s = json_object_to_json_string(o); + json_object_put(o); + return s + "\n"; +} + +// --------------------------------------------------------------------------- +// Command handlers +// --------------------------------------------------------------------------- + +static std::string handle_list() { + json_object *arr = json_object_new_array(); + for (int i = 0; i < N_DEMOS; i++) { + json_object *d = json_object_new_object(); + json_object_object_add(d, "name", json_object_new_string(DEMOS[i].name)); + json_object_object_add(d, "description", json_object_new_string(DEMOS[i].description)); + json_object_object_add(d, "type", + json_object_new_string(DEMOS[i].is_edge_ai ? "edge-ai" : "dsp-compute")); + json_object_array_add(arr, d); + } + return resp_ok(arr); +} + +static std::string handle_status() { + json_object *d = json_object_new_object(); + if (g_child_pid > 0) { + json_object_object_add(d, "running", json_object_new_string(g_active_demo)); + json_object_object_add(d, "pid", json_object_new_int(static_cast(g_child_pid))); + } else { + json_object_object_add(d, "running", nullptr); + json_object_object_add(d, "pid", json_object_new_int(-1)); + } + return resp_ok(d); +} + +static std::string handle_stop() { + if (g_child_pid < 0) return resp_err("no demo is running"); + stop_child(); + return resp_ok(); +} + +static std::string handle_preload() { + if (g_child_pid > 0) + return resp_err(std::string("stop '") + g_active_demo + "' before preloading"); + + syslog(LOG_INFO, "Running TVM model preload"); + int rc = run_sync({PRELOAD_EXE, "--preload"}); + if (rc != 0) + return resp_err("preload failed with exit code " + std::to_string(rc)); + + syslog(LOG_INFO, "TVM model preload completed"); + json_object *d = json_object_new_object(); + json_object_object_add(d, "message", json_object_new_string("TVM model preload completed")); + return resp_ok(d); +} + +static std::string handle_run(const std::string &name, const std::vector &extra) { + if (g_child_pid > 0) + return resp_err(std::string("demo '") + g_active_demo + "' is already running - stop it first"); + + const DemoEntry *demo = nullptr; + for (int i = 0; i < N_DEMOS; i++) { + if (name == DEMOS[i].name) { demo = &DEMOS[i]; break; } + } + if (!demo) return resp_err("unknown demo: " + name); + + // For edge-ai: silently preload if the TVM model cache doesn't exist + if (demo->is_edge_ai && !file_exists(TVM_CACHE)) { + syslog(LOG_INFO, "TVM model cache not found, running preload before '%s'", name.c_str()); + int rc = run_sync({PRELOAD_EXE, "--preload"}); + if (rc != 0) + syslog(LOG_WARNING, "Auto-preload failed (rc=%d), proceeding anyway", rc); + } + + std::vector argv = {demo->executable}; + argv.insert(argv.end(), extra.begin(), extra.end()); + + pid_t pid = launch_async(argv); + if (pid < 0) return resp_err("failed to fork process"); + + g_child_pid = pid; + strncpy(g_active_demo, name.c_str(), sizeof(g_active_demo) - 1); + g_active_demo[sizeof(g_active_demo) - 1] = '\0'; + g_invalidate_on_exit = demo->is_dsp_compute; + + syslog(LOG_INFO, "Started demo '%s' pid=%d (invalidate_tvm_on_exit=%s)", + name.c_str(), (int)pid, demo->is_dsp_compute ? "yes" : "no"); + + json_object *d = json_object_new_object(); + json_object_object_add(d, "demo", json_object_new_string(name.c_str())); + json_object_object_add(d, "pid", json_object_new_int(static_cast(pid))); + return resp_ok(d); +} + +// --------------------------------------------------------------------------- +// Command dispatch +// --------------------------------------------------------------------------- + +static std::string dispatch(const char *line) { + json_object *root = json_tokener_parse(line); + if (!root) return resp_err("invalid JSON"); + + json_object *cmd_obj = nullptr; + if (!json_object_object_get_ex(root, "cmd", &cmd_obj)) { + json_object_put(root); + return resp_err("missing 'cmd' field"); + } + std::string cmd = json_object_get_string(cmd_obj); + + std::string result; + if (cmd == "list") { result = handle_list(); } + else if (cmd == "status") { result = handle_status(); } + else if (cmd == "stop") { result = handle_stop(); } + else if (cmd == "preload") { result = handle_preload(); } + else if (cmd == "quit") { g_quit = 1; result = resp_ok(); } + else if (cmd == "run") { + json_object *name_obj = nullptr; + if (!json_object_object_get_ex(root, "demo", &name_obj)) { + result = resp_err("'run' requires a 'demo' field"); + } else { + std::vector args; + json_object *arr = nullptr; + if (json_object_object_get_ex(root, "args", &arr)) { + int n = json_object_array_length(arr); + args.reserve(static_cast(n)); + for (int i = 0; i < n; i++) { + auto *el = json_object_array_get_idx(arr, i); + args.emplace_back(json_object_get_string(el)); + } + } + result = handle_run(json_object_get_string(name_obj), args); + } + } else { + result = resp_err("unknown command: " + cmd); + } + + json_object_put(root); + return result; +} + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +int main() { + // Signal setup + struct sigaction sa{}; + sigemptyset(&sa.sa_mask); + sa.sa_handler = on_signal; + sigaction(SIGTERM, &sa, nullptr); + sigaction(SIGINT, &sa, nullptr); + sa.sa_handler = on_sigchld; + sa.sa_flags = SA_NOCLDSTOP; + sigaction(SIGCHLD, &sa, nullptr); + + openlog("demo-manager", LOG_PID | LOG_CONS, LOG_DAEMON); + syslog(LOG_INFO, "demo-manager daemon starting"); + + // PID file + if (FILE *f = fopen(PID_FILE, "w")) { + fprintf(f, "%d\n", getpid()); + fclose(f); + } + + // Create Unix domain socket + int srv = socket(AF_UNIX, SOCK_STREAM, 0); + if (srv < 0) { syslog(LOG_ERR, "socket: %s", strerror(errno)); return 1; } + + unlink(SOCKET_PATH); + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_PATH, sizeof(addr.sun_path) - 1); + + if (bind(srv, reinterpret_cast(&addr), sizeof(addr)) < 0) { + syslog(LOG_ERR, "bind %s: %s", SOCKET_PATH, strerror(errno)); + return 1; + } + chmod(SOCKET_PATH, 0666); + listen(srv, 8); + + syslog(LOG_INFO, "listening on %s", SOCKET_PATH); + + // Main event loop + while (!g_quit) { + if (g_child_done) reap_child(); + + pollfd pfd{srv, POLLIN, 0}; + int r = poll(&pfd, 1, 500); // 500 ms so signals are checked regularly + if (r < 0) { + if (errno == EINTR) continue; + syslog(LOG_ERR, "poll: %s", strerror(errno)); + break; + } + if (!r || !(pfd.revents & POLLIN)) continue; + + int cli = accept(srv, nullptr, nullptr); + if (cli < 0) { if (errno != EINTR) syslog(LOG_WARNING, "accept: %s", strerror(errno)); continue; } + + char buf[4096] = {}; + ssize_t n = read(cli, buf, sizeof(buf) - 1); + if (n > 0) { + // Strip trailing whitespace / newlines + while (n > 0 && static_cast(buf[n - 1]) <= ' ') buf[--n] = '\0'; + syslog(LOG_DEBUG, "cmd: %s", buf); + std::string resp = dispatch(buf); + write(cli, resp.c_str(), resp.size()); + } + close(cli); + } + + syslog(LOG_INFO, "shutting down"); + stop_child(); // stop any running demo and invalidate cache if needed + + close(srv); + unlink(SOCKET_PATH); + unlink(PID_FILE); + closelog(); + return 0; +} diff --git a/example/edge-ai/CMakeLists.txt b/example/edge-ai/CMakeLists.txt index 36177fd..b37a168 100644 --- a/example/edge-ai/CMakeLists.txt +++ b/example/edge-ai/CMakeLists.txt @@ -71,9 +71,17 @@ message(STATUS "Using TVM runtime: ${TVM_RUNTIME_LIB}") find_package(PkgConfig REQUIRED) pkg_check_modules(JSON_C REQUIRED json-c) +pkg_get_variable(SYSTEMD_SYSTEM_UNITDIR systemd systemdsystemunitdir) +if(NOT SYSTEMD_SYSTEM_UNITDIR) + set(SYSTEMD_SYSTEM_UNITDIR "${CMAKE_INSTALL_PREFIX}/lib/systemd/system") +endif() + # Add json-c include directory target_include_directories(rpmsg_inference_example PRIVATE ${JSON_C_INCLUDE_DIRS}) +# Find readline library for interactive mode with command line editing +find_library(READLINE_LIB readline REQUIRED) + # Find audio libraries for WAV file processing and playback find_library(SNDFILE_LIB sndfile REQUIRED) find_library(ALSA_LIB asound REQUIRED) @@ -83,6 +91,7 @@ target_link_libraries(rpmsg_inference_example ${TVM_RUNTIME_LIB} ti_rpmsg_dma ${JSON_C_LIBRARIES} + ${READLINE_LIB} ${SNDFILE_LIB} ${ALSA_LIB} pthread @@ -92,15 +101,54 @@ target_link_libraries(rpmsg_inference_example # Install target install(TARGETS rpmsg_inference_example DESTINATION ${CMAKE_INSTALL_BINDIR}) +# ── Model loader daemon ────────────────────────────────────────────────────── +# Loads Module::LoadFromFile once at boot and serves inference requests over a +# Unix domain socket so that rpmsg_inference_example skips artifact loading. + +add_executable(tvm_model_daemon + src/tvm_model_daemon.cpp + src/tvm_inference_client.cpp +) + +target_include_directories(tvm_model_daemon PRIVATE + include + ${TVM_ROOT}/include + ${TVM_ROOT}/3rdparty/dlpack/include + ${TVM_ROOT}/3rdparty/dmlc-core/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../library/include +) + +target_compile_definitions(tvm_model_daemon PRIVATE + DMLC_USE_LOGGING_LIBRARY= +) + +target_link_libraries(tvm_model_daemon + ${TVM_RUNTIME_LIB} + ti_rpmsg_dma + dl + pthread +) + +install(TARGETS tvm_model_daemon DESTINATION ${CMAKE_INSTALL_BINDIR}) + # Install JSON pipeline configuration files to /usr/share/tvm_inference/json/ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_stft_istft.json ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_tvm_inference.json ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_audio_enhancement.json -DESTINATION /usr/share/tvm_inference/json) +DESTINATION ${CMAKE_INSTALL_DATADIR}/tvm_inference/json) # Install input files to /usr/share/tvm_inference/input/ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/input_audio/input_audio.wav ${CMAKE_CURRENT_SOURCE_DIR}/artifacts_bin/gcrn_fixed_input.bin - DESTINATION /usr/share/tvm_inference/input) + DESTINATION ${CMAKE_INSTALL_DATADIR}/tvm_inference/input) + +# Install systemd services +install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/tvm-model-preload.service + ${CMAKE_CURRENT_SOURCE_DIR}/tvm-model-daemon.service + DESTINATION ${SYSTEMD_SYSTEM_UNITDIR}) + +# Create the model cache directory at install time +install(DIRECTORY DESTINATION /var/lib/tvm_inference) diff --git a/example/edge-ai/include/pipeline_manager.h b/example/edge-ai/include/pipeline_manager.h index 80a70c1..20fee5f 100644 --- a/example/edge-ai/include/pipeline_manager.h +++ b/example/edge-ai/include/pipeline_manager.h @@ -64,10 +64,16 @@ class PipelineManager { State() : input_type(InputType::UNKNOWN) {} }; + // Path where the currently-loaded model artifacts path is persisted across runs + static constexpr const char* MODEL_CACHE_FILE = "/var/lib/tvm_inference/loaded_model"; + // Default artifacts loaded at boot via --preload + static constexpr const char* DEFAULT_ARTIFACTS_PATH = "/usr/share/tvm_inference/artifacts/"; + PipelineManager(); ~PipelineManager(); bool initialize(); int run_from_json_file(const std::string& json_file_path); + int preload_default_model(); void set_debug(bool enable) { debug_ = enable; } private: @@ -79,6 +85,10 @@ class PipelineManager { bool validateConfiguration(); bool loadPipelineFromJson(const std::string& json_content); + + // Model cache: read/write the artifacts path persisted on disk + static std::string read_model_cache(); + static bool write_model_cache(const std::string& artifacts_path); }; #endif // PIPELINE_MANAGER_H diff --git a/example/edge-ai/include/tvm_daemon_proto.h b/example/edge-ai/include/tvm_daemon_proto.h new file mode 100644 index 0000000..5dcd2c0 --- /dev/null +++ b/example/edge-ai/include/tvm_daemon_proto.h @@ -0,0 +1,37 @@ +/* + * tvm_daemon_proto.h — Wire protocol shared by tvm_model_daemon and TvmInferenceClient. + * + * Layout (all fields native-endian / little-endian on ARM): + * + * Client → Daemon: Header{MAGIC, PING, 0} + * Daemon → Client: Header{MAGIC, PONG, 0} + * + * Client → Daemon: Header{MAGIC, INFER_REQ, n_bytes} + n_bytes of float32 input + * Daemon → Client: Header{MAGIC, INFER_RESP, n_bytes} + n_bytes of float32 output + * Daemon → Client: Header{MAGIC, ERROR_RESP, n_bytes} + n_bytes of UTF-8 error string + */ + +#pragma once + +#include + +namespace TvmDaemon { + +static constexpr const char* SOCKET_PATH = "/var/run/tvm-inference.sock"; +static constexpr uint32_t MAGIC = 0x544D5644u; /* 'TMVD' */ + +enum class MsgType : uint32_t { + PING = 0, + PONG = 1, + INFER_REQ = 2, + INFER_RESP = 3, + ERROR_RESP = 4, +}; + +struct Header { + uint32_t magic; + uint32_t type; /* MsgType cast to uint32_t */ + uint32_t len; /* payload bytes following this header (0 for PING/PONG) */ +}; + +} // namespace TvmDaemon diff --git a/example/edge-ai/include/tvm_inference_client.h b/example/edge-ai/include/tvm_inference_client.h index f6d7504..5cc2334 100644 --- a/example/edge-ai/include/tvm_inference_client.h +++ b/example/edge-ai/include/tvm_inference_client.h @@ -14,6 +14,9 @@ class PackedFunc; } } +// Forward declaration — full definition in tvm_daemon_proto.h (included by .cpp) +namespace TvmDaemon { struct Header; } + class TvmInferenceClient { private: std::string artifacts_path_; @@ -31,7 +34,12 @@ class TvmInferenceClient { // Model info std::string input_name_; std::vector input_shape_; - int dma_buffer_fd_{-1}; + int dma_input_fd_{-1}; + int dma_output_fd_{-1}; + + // Daemon client state + int daemon_fd_{-1}; /* Unix socket fd when using daemon; -1 = local mode */ + bool daemon_skip_{false}; /* true = never try daemon (set by daemon itself) */ public: TvmInferenceClient(); @@ -47,6 +55,7 @@ class TvmInferenceClient { bool run_inference(const std::vector& input_data, std::vector& output_data, const std::vector& input_shape); + bool run_inference(const float* input, float* output, size_t count); bool run_inference(std::vector& dint_data, std::vector& inter_data, size_t data_size); bool run_inference(const std::string& bin_path); @@ -56,14 +65,24 @@ class TvmInferenceClient { const std::vector& get_input_shape() const { return input_shape_; } void set_input_shape(const std::vector& shape) { input_shape_ = shape; } void set_input_name(const std::string& name) { input_name_ = name; } - void set_dma_buffer_fd(int descriptor) noexcept { dma_buffer_fd_ = descriptor; } + void set_input_dma_fd(int fd) noexcept { dma_input_fd_ = fd; } + void set_output_dma_fd(int fd) noexcept { dma_output_fd_ = fd; } + + /* Prevent daemon auto-connect — must be called before initialize(). + * Used by tvm_model_daemon to avoid recursion into itself. */ + void disable_daemon() noexcept { daemon_skip_ = true; } + bool is_daemon_mode() const noexcept { return daemon_fd_ >= 0; } private: // Helper methods bool load_artifacts(); void process_output_data(); - bool synchronize_dma_buffer(int operation) const noexcept; + bool synchronize_dma_buffer(int fd, int operation) const noexcept; std::string load_json_file(const std::string& path); + + // Daemon client helpers + bool try_daemon_connect(); + bool run_via_daemon(const float* input, float* output, size_t count); }; #endif // TVM_INFERENCE_CLIENT_H diff --git a/example/edge-ai/src/main.cpp b/example/edge-ai/src/main.cpp index fee8d30..5b38b19 100644 --- a/example/edge-ai/src/main.cpp +++ b/example/edge-ai/src/main.cpp @@ -35,6 +35,7 @@ void print_usage(std::string_view program) << "Usage:\n" << " " << program << " Run a JSON pipeline\n" << " " << program << " --debug Enable per-batch logs\n" + << " " << program << " --preload Load default model into C7x (run at boot)\n" << " " << program << " --version Show version and build info\n" << " " << program << " --help Show this help\n\n" << "Examples:\n" @@ -49,6 +50,7 @@ int main(int argc, char* argv[]) try { std::string json_file; bool debug = false; + bool preload = false; for (int index = 1; index < argc; ++index) { const std::string_view argument{argv[index]}; @@ -64,6 +66,10 @@ int main(int argc, char* argv[]) debug = true; continue; } + if (argument == "--preload") { + preload = true; + continue; + } if (argument.rfind("--", 0) == 0) { std::cerr << "[App] Unknown argument: " << argument << '\n'; print_usage(argv[0]); @@ -76,7 +82,7 @@ int main(int argc, char* argv[]) json_file = argument; } - if (json_file.empty()) { + if (!preload && json_file.empty()) { std::cerr << "[App] Error: A pipeline JSON file is required\n"; print_usage(argv[0]); return EXIT_FAILURE; @@ -90,6 +96,10 @@ int main(int argc, char* argv[]) PipelineManager application; application.set_debug(debug); + + if (preload) + return application.preload_default_model(); + const int exit_code = application.run_from_json_file(json_file); std::cout << "[App] Application exited with code " << exit_code << '\n'; return exit_code; diff --git a/example/edge-ai/src/pipeline_manager.cpp b/example/edge-ai/src/pipeline_manager.cpp index e3e8705..50958a1 100644 --- a/example/edge-ai/src/pipeline_manager.cpp +++ b/example/edge-ai/src/pipeline_manager.cpp @@ -13,6 +13,63 @@ extern "C" { #include } +// ─── Model cache ────────────────────────────────────────────────────────────── + +std::string PipelineManager::read_model_cache() +{ + std::ifstream f(MODEL_CACHE_FILE); + if (!f.is_open()) + return {}; + std::string path; + std::getline(f, path); + return path; +} + +bool PipelineManager::write_model_cache(const std::string& artifacts_path) +{ + std::filesystem::create_directories( + std::filesystem::path(MODEL_CACHE_FILE).parent_path()); + std::ofstream f(MODEL_CACHE_FILE, std::ios::trunc); + if (!f.is_open()) { + std::cerr << "[App] Warning: cannot write model cache: " << MODEL_CACHE_FILE << std::endl; + return false; + } + f << artifacts_path << '\n'; + return true; +} + +// ─── Preload default model at boot ──────────────────────────────────────────── + +int PipelineManager::preload_default_model() +{ + if (!initialize()) + return -1; + + const std::string artifacts = DEFAULT_ARTIFACTS_PATH; + std::cout << "[App] Preloading default model: " << artifacts << std::endl; + + const std::string cached = read_model_cache(); + if (cached == artifacts) { + std::cout << "[App] Model already loaded (cache matches), nothing to do." << std::endl; + return 0; + } + + tvm_client_ = std::make_shared(); + if (!tvm_client_->initialize(artifacts)) { + std::cerr << "[App] Failed to load default model from: " << artifacts << std::endl; + return -1; + } + tvm_client_->set_input_shape({1, 2, 401, 161}); + + if (!write_model_cache(artifacts)) + std::cerr << "[App] Warning: model loaded but cache write failed" << std::endl; + + std::cout << "[App] Default model loaded and cached." << std::endl; + return 0; +} + +// ────────────────────────────────────────────────────────────────────────────── + PipelineManager::PipelineManager() : initialized_(false) { diff --git a/example/edge-ai/src/tvm_inference_client.cpp b/example/edge-ai/src/tvm_inference_client.cpp index 02b6110..62a7009 100644 --- a/example/edge-ai/src/tvm_inference_client.cpp +++ b/example/edge-ai/src/tvm_inference_client.cpp @@ -1,4 +1,5 @@ #include "tvm_inference_client.h" +#include "tvm_daemon_proto.h" #include #include @@ -10,6 +11,9 @@ #include #include +#include +#include +#include // TVM runtime includes #include @@ -26,6 +30,26 @@ using namespace tvm::runtime; namespace { +static bool sock_write_all(int fd, const void* buf, size_t len) { + const auto* p = static_cast(buf); + while (len > 0) { + ssize_t n = ::write(fd, p, len); + if (n <= 0) return false; + p += static_cast(n); len -= static_cast(n); + } + return true; +} + +static bool sock_read_all(int fd, void* buf, size_t len) { + auto* p = static_cast(buf); + while (len > 0) { + ssize_t n = ::read(fd, p, len); + if (n <= 0) return false; + p += static_cast(n); len -= static_cast(n); + } + return true; +} + template size_t tensor_element_count(const Tensor* tensor) { @@ -45,9 +69,8 @@ size_t tensor_element_count(const Tensor* tensor) } // namespace -bool TvmInferenceClient::synchronize_dma_buffer(int operation) const noexcept { - // DMA synchronization is optional until a caller supplies a valid fd. - return dma_buffer_fd_ < 0 || dmabuf_sync(dma_buffer_fd_, operation) == 0; +bool TvmInferenceClient::synchronize_dma_buffer(int fd, int operation) const noexcept { + return fd < 0 || dmabuf_sync(fd, operation) == 0; } TvmInferenceClient::TvmInferenceClient() : initialized_(false) { @@ -66,6 +89,15 @@ bool TvmInferenceClient::initialize(const std::string& artifacts_path) { artifacts_path_ = artifacts_path; + /* Fast path: delegate to persistent daemon — skips Module::LoadFromFile */ + if (!daemon_skip_ && try_daemon_connect()) { + initialized_ = true; + std::cout << "[TVM] Connected to model daemon at " + << TvmDaemon::SOCKET_PATH << std::endl; + return true; + } + + /* Fall back to loading artifacts locally */ try { if (!load_artifacts()) return false; @@ -79,7 +111,83 @@ bool TvmInferenceClient::initialize(const std::string& artifacts_path) { } } +bool TvmInferenceClient::try_daemon_connect() { + int fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) return false; + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::strncpy(addr.sun_path, TvmDaemon::SOCKET_PATH, sizeof(addr.sun_path) - 1); + + if (::connect(fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { + ::close(fd); + return false; + } + + /* Ping/pong: confirm daemon is alive and ready */ + TvmDaemon::Header ping{TvmDaemon::MAGIC, + static_cast(TvmDaemon::MsgType::PING), 0}; + TvmDaemon::Header pong{}; + if (!sock_write_all(fd, &ping, sizeof(ping)) || + !sock_read_all(fd, &pong, sizeof(pong)) || + pong.magic != TvmDaemon::MAGIC || + pong.type != static_cast(TvmDaemon::MsgType::PONG)) { + ::close(fd); + return false; + } + + daemon_fd_ = fd; + return true; +} + +bool TvmInferenceClient::run_via_daemon(const float* input, float* output, size_t count) { + const uint32_t in_bytes = static_cast(count * sizeof(float)); + TvmDaemon::Header req{TvmDaemon::MAGIC, + static_cast(TvmDaemon::MsgType::INFER_REQ), + in_bytes}; + + if (!sock_write_all(daemon_fd_, &req, sizeof(req)) || + !sock_write_all(daemon_fd_, input, in_bytes)) { + std::cerr << "[TVM] Daemon: failed to send request\n"; + return false; + } + + TvmDaemon::Header resp{}; + if (!sock_read_all(daemon_fd_, &resp, sizeof(resp)) || + resp.magic != TvmDaemon::MAGIC) { + std::cerr << "[TVM] Daemon: invalid response header\n"; + return false; + } + + if (resp.type == static_cast(TvmDaemon::MsgType::ERROR_RESP)) { + std::vector msg(resp.len + 1, '\0'); + sock_read_all(daemon_fd_, msg.data(), resp.len); + std::cerr << "[TVM] Daemon error: " << msg.data() << '\n'; + return false; + } + + if (resp.type != static_cast(TvmDaemon::MsgType::INFER_RESP)) { + std::cerr << "[TVM] Daemon: unexpected response type " << resp.type << '\n'; + return false; + } + + const size_t out_floats = resp.len / sizeof(float); + if (out_floats > count) { + std::cerr << "[TVM] Daemon: output too large (" << out_floats + << " > " << count << ")\n"; + return false; + } + + if (!sock_read_all(daemon_fd_, output, resp.len)) { + std::cerr << "[TVM] Daemon: failed to read output\n"; + return false; + } + + return true; +} + void TvmInferenceClient::cleanup() { + if (daemon_fd_ >= 0) { ::close(daemon_fd_); daemon_fd_ = -1; } get_output_.reset(); run_.reset(); set_input_.reset(); @@ -167,7 +275,7 @@ bool TvmInferenceClient::load_artifacts() { // Load parameters into executor auto load_params = graph_executor_->GetFunction("load_params"); - if (load_params == nullptr) + if (load_params == nullptr) return false; TVMByteArray param_array; param_array.data = reinterpret_cast(param_data.data()); @@ -206,6 +314,11 @@ bool TvmInferenceClient::run_inference(const std::vector& input_data, return false; } + if (daemon_fd_ >= 0) { + output_data.resize(input_data.size()); + return run_via_daemon(input_data.data(), output_data.data(), input_data.size()); + } + try { const size_t shape_elements = std::accumulate( input_shape.begin(), input_shape.end(), size_t{1}, @@ -220,6 +333,9 @@ bool TvmInferenceClient::run_inference(const std::vector& input_data, const auto start = std::chrono::steady_clock::now(); + if (!synchronize_dma_buffer(dma_input_fd_, DMA_BUF_SYNC_START)) + throw std::runtime_error{"Failed to sync input DMA buffer for CPU access"}; + NDArray input_array = NDArray::Empty(input_shape, DLDataType{kDLFloat, 32, 1}, {kDLCPU, 0}); input_array.CopyFromBytes(input_data.data(), input_data.size() * sizeof(float)); if (input_name_.empty()) @@ -233,6 +349,92 @@ bool TvmInferenceClient::run_inference(const std::vector& input_data, output_data.resize(output_elements); out.CopyToBytes(output_data.data(), output_elements * sizeof(float)); + if (!synchronize_dma_buffer(dma_output_fd_, DMA_BUF_SYNC_END)) + throw std::runtime_error{"Failed to complete DMA buffer cache synchronization"}; + + const auto end = std::chrono::steady_clock::now(); + double ms = std::chrono::duration_cast(end - start).count() / 1000.0; + + std::cout << "[TVM] Inference done in " << ms << " ms, output: " + << output_elements << " floats" << std::endl; + return true; + + } catch (const std::exception& e) { + std::cerr << "[TVM] Inference failed: " << e.what() << std::endl; + return false; + } +} + +bool TvmInferenceClient::run_inference(const float* input, float* output, size_t count) { + if (!initialized_) { + std::cerr << "[TVM] Not initialized" << std::endl; + return false; + } + if (!input || count == 0 || !output) { + std::cerr << "[TVM] Invalid input/output pointer or count" << std::endl; + return false; + } + + /* Daemon mode — send over socket instead of running TVM locally */ + if (daemon_fd_ >= 0) { + const auto t0 = std::chrono::steady_clock::now(); + const bool ok = run_via_daemon(input, output, count); + const double ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count() / 1000.0; + if (ok) std::cout << "[TVM] Daemon inference done in " << ms + << " ms, " << count << " floats\n"; + return ok; + } + + std::vector shape; + if (input_shape_.empty()) + shape = {static_cast(count)}; + else + shape.assign(input_shape_.begin(), input_shape_.end()); + + try { + const size_t shape_elements = std::accumulate( + shape.begin(), shape.end(), size_t{1}, + [](size_t total, int64_t extent) { + if (extent <= 0 || total > std::numeric_limits::max() / + static_cast(extent)) + throw std::overflow_error{"Invalid TVM input shape"}; + return total * static_cast(extent); + }); + if (shape_elements != count) + throw std::runtime_error{"TVM input shape does not match the input count"}; + + const auto start = std::chrono::steady_clock::now(); + + if (!synchronize_dma_buffer(dma_input_fd_, DMA_BUF_SYNC_START)) + throw std::runtime_error{"Failed to sync input DMA buffer for CPU access"}; + + NDArray input_array = NDArray::Empty(shape, DLDataType{kDLFloat, 32, 1}, {kDLCPU, 0}); + input_array.CopyFromBytes(input, count * sizeof(float)); + if (input_name_.empty()) + (*set_input_)(0, input_array); + else + (*set_input_)(String{input_name_}, input_array); + + if (!synchronize_dma_buffer(dma_input_fd_, DMA_BUF_SYNC_END)) + throw std::runtime_error{"Failed to end input DMA buffer sync"}; + + (*run_)(); + + if (!synchronize_dma_buffer(dma_output_fd_, DMA_BUF_SYNC_START)) + throw std::runtime_error{"Failed to sync output DMA buffer for CPU access"}; + + NDArray out = (*get_output_)(0); + const size_t output_elements = tensor_element_count(out.operator->()); + if (output_elements > count) + throw std::runtime_error{"Output buffer too small: need " + + std::to_string(output_elements) + " floats, got " + + std::to_string(count)}; + out.CopyToBytes(output, output_elements * sizeof(float)); + + if (!synchronize_dma_buffer(dma_output_fd_, DMA_BUF_SYNC_END)) + throw std::runtime_error{"Failed to complete output DMA buffer sync"}; + const auto end = std::chrono::steady_clock::now(); double ms = std::chrono::duration_cast(end - start).count() / 1000.0; @@ -287,5 +489,5 @@ bool TvmInferenceClient::run_inference(const std::string& bin_path) { std::cout << "[TVM] Loaded " << num_floats << " floats from " << bin_path << std::endl; return run_inference(input_data_, output_data_, - std::vector{static_cast(num_floats)}); + std::vector{static_cast(num_floats)}); } diff --git a/example/edge-ai/src/tvm_model_daemon.cpp b/example/edge-ai/src/tvm_model_daemon.cpp new file mode 100644 index 0000000..6c0d445 --- /dev/null +++ b/example/edge-ai/src/tvm_model_daemon.cpp @@ -0,0 +1,182 @@ +/* + * tvm_model_daemon.cpp — Persistent TVM model loader daemon + * + * Calls Module::LoadFromFile once at startup, then serves inference requests + * from rpmsg_inference_example instances over a Unix domain socket. Each demo + * run connects, issues one INFER_REQ per chunk, then disconnects. The model + * stays resident for the lifetime of this process. + * + * Usage: + * tvm_model_daemon [--artifacts /path/to/artifacts/dir] + * + * Managed by tvm-model-daemon.service (started at boot, before the webserver). + */ + +#include "tvm_inference_client.h" +#include "tvm_daemon_proto.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr const char* DEFAULT_ARTIFACTS = "/usr/share/tvm_inference/artifacts/"; + +namespace { + +volatile sig_atomic_t g_running = 1; +int g_server_fd = -1; + +void signal_handler(int) { + g_running = 0; + if (g_server_fd >= 0) { ::close(g_server_fd); g_server_fd = -1; } +} + +bool write_all(int fd, const void* buf, size_t len) { + const auto* p = static_cast(buf); + while (len > 0) { + ssize_t n = ::write(fd, p, len); + if (n <= 0) return false; + p += static_cast(n); len -= static_cast(n); + } + return true; +} + +bool read_all(int fd, void* buf, size_t len) { + auto* p = static_cast(buf); + while (len > 0) { + ssize_t n = ::read(fd, p, len); + if (n <= 0) return false; + p += static_cast(n); len -= static_cast(n); + } + return true; +} + +bool send_hdr(int fd, TvmDaemon::MsgType type, uint32_t payload_len) { + TvmDaemon::Header h{TvmDaemon::MAGIC, static_cast(type), payload_len}; + return write_all(fd, &h, sizeof(h)); +} + +/* Serve a connected client until it disconnects or an error occurs. + * Handles PING and repeated INFER_REQ messages on a single connection. */ +void handle_client(int cfd, TvmInferenceClient& tvm) { + TvmDaemon::Header hdr{}; + while (true) { + if (!read_all(cfd, &hdr, sizeof(hdr))) break; + if (hdr.magic != TvmDaemon::MAGIC) { + std::cerr << "[daemon] Bad magic 0x" << std::hex << hdr.magic + << std::dec << " — closing\n"; + break; + } + + if (hdr.type == static_cast(TvmDaemon::MsgType::PING)) { + send_hdr(cfd, TvmDaemon::MsgType::PONG, 0); + continue; + } + + if (hdr.type != static_cast(TvmDaemon::MsgType::INFER_REQ) || + hdr.len == 0 || hdr.len % sizeof(float) != 0) { + const std::string err = "unexpected message type or bad payload size"; + send_hdr(cfd, TvmDaemon::MsgType::ERROR_RESP, + static_cast(err.size())); + write_all(cfd, err.data(), err.size()); + break; + } + + const size_t n_floats = hdr.len / sizeof(float); + std::vector input(n_floats), output; + + if (!read_all(cfd, input.data(), hdr.len)) { + std::cerr << "[daemon] Failed to read input payload\n"; + break; + } + + const auto t0 = std::chrono::steady_clock::now(); + const bool ok = tvm.run_inference(input, output); + const double ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count() / 1000.0; + + if (!ok) { + const std::string err = "inference failed"; + send_hdr(cfd, TvmDaemon::MsgType::ERROR_RESP, + static_cast(err.size())); + write_all(cfd, err.data(), err.size()); + continue; + } + + std::cout << "[daemon] Inference " << ms << " ms (" + << output.size() << " floats out)\n"; + + const uint32_t out_bytes = static_cast(output.size() * sizeof(float)); + if (!send_hdr(cfd, TvmDaemon::MsgType::INFER_RESP, out_bytes) || + !write_all(cfd, output.data(), out_bytes)) { + std::cerr << "[daemon] Failed to send response\n"; + break; + } + } + ::close(cfd); +} + +} // namespace + +int main(int argc, char* argv[]) { + std::string artifacts = DEFAULT_ARTIFACTS; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--artifacts" && i + 1 < argc) + artifacts = argv[++i]; + } + + std::cout << "[daemon] Loading TVM artifacts from: " << artifacts << '\n'; + + TvmInferenceClient tvm; + tvm.disable_daemon(); /* must not try to connect to itself */ + tvm.set_input_shape({1, 2, 401, 161}); + + if (!tvm.initialize(artifacts)) { + std::cerr << "[daemon] Failed to load model — aborting\n"; + return 1; + } + std::cout << "[daemon] Model ready. Listening on " + << TvmDaemon::SOCKET_PATH << '\n'; + + std::signal(SIGINT, signal_handler); + std::signal(SIGTERM, signal_handler); + std::signal(SIGPIPE, SIG_IGN); /* suppress broken-pipe crashes */ + + ::unlink(TvmDaemon::SOCKET_PATH); + g_server_fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (g_server_fd < 0) { perror("socket"); return 1; } + + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::strncpy(addr.sun_path, TvmDaemon::SOCKET_PATH, sizeof(addr.sun_path) - 1); + + if (::bind(g_server_fd, reinterpret_cast(&addr), sizeof(addr)) < 0) { + perror("bind"); return 1; + } + ::chmod(TvmDaemon::SOCKET_PATH, 0660); + + if (::listen(g_server_fd, 4) < 0) { perror("listen"); return 1; } + std::cout << "[daemon] Ready\n"; + + while (g_running) { + int cfd = ::accept(g_server_fd, nullptr, nullptr); + if (cfd < 0) { + if (g_running) perror("accept"); + break; + } + std::cout << "[daemon] Client connected\n"; + handle_client(cfd, tvm); + std::cout << "[daemon] Client done\n"; + } + + ::unlink(TvmDaemon::SOCKET_PATH); + std::cout << "[daemon] Shut down\n"; + return 0; +} diff --git a/example/edge-ai/tvm-model-daemon.service b/example/edge-ai/tvm-model-daemon.service new file mode 100644 index 0000000..dd1a7f6 --- /dev/null +++ b/example/edge-ai/tvm-model-daemon.service @@ -0,0 +1,17 @@ +[Unit] +Description=TVM Model Loader Daemon (GCRN Speech Enhancement) +# Must be ready before the webserver so the first demo run hits the daemon +After=sysinit.target local-fs.target +Wants=local-fs.target + +[Service] +Type=simple +ExecStart=/usr/bin/tvm_model_daemon +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal +# Socket lives in /var/run — always present, no pre-create needed + +[Install] +WantedBy=multi-user.target diff --git a/example/edge-ai/tvm-model-preload.service b/example/edge-ai/tvm-model-preload.service new file mode 100644 index 0000000..310f8b8 --- /dev/null +++ b/example/edge-ai/tvm-model-preload.service @@ -0,0 +1,17 @@ +[Unit] +Description=Preload default TVM model into C7x DSP +# Run after remoteproc firmware is up and filesystem is mounted +After=sysinit.target local-fs.target +Wants=local-fs.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/rpmsg_inference_example --preload +StandardOutput=journal +StandardError=journal +# Create the cache directory with correct permissions before first run +ExecStartPre=/bin/mkdir -p /var/lib/tvm_inference + +[Install] +WantedBy=multi-user.target From 31693ea02d6fcb17744de75d7789d8f32f75323e Mon Sep 17 00:00:00 2001 From: Paresh Bhagat Date: Fri, 21 Aug 2026 16:36:15 +0530 Subject: [PATCH 4/7] example: edge-ai: Update STFT/ISTFT message structure and JSON params - Rename stft_process_msg -> stft_istft_msg, remove redundant graph_id with selected_model. - Add selected_model to STFT/ISTFT stage parameters in JSON files Signed-off-by: Paresh Bhagat --- .../pipeline_audio_enhancement.json | 2 ++ .../json_files/pipeline_stft_istft.json | 2 ++ example/edge-ai/src/dsp_task_client.cpp | 27 ++++++++++--------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/example/edge-ai/json_files/pipeline_audio_enhancement.json b/example/edge-ai/json_files/pipeline_audio_enhancement.json index 54c860b..608ed52 100644 --- a/example/edge-ai/json_files/pipeline_audio_enhancement.json +++ b/example/edge-ai/json_files/pipeline_audio_enhancement.json @@ -13,6 +13,7 @@ "service": "generic", "message_type": "C7X_MSG_STFT_ANALYZE", "parameters": { + "selected_model": 2, "hop_size": 160, "model_elems": 322, "total_frames": 401, @@ -50,6 +51,7 @@ "service": "generic", "message_type": "C7X_MSG_ISTFT_SYNTHESIZE", "parameters": { + "selected_model": 2, "hop_size": 160, "model_elems": 322, "total_frames": 401, diff --git a/example/edge-ai/json_files/pipeline_stft_istft.json b/example/edge-ai/json_files/pipeline_stft_istft.json index fb848bc..9a5d08a 100644 --- a/example/edge-ai/json_files/pipeline_stft_istft.json +++ b/example/edge-ai/json_files/pipeline_stft_istft.json @@ -12,6 +12,7 @@ "service": "generic", "message_type": "C7X_MSG_STFT_ANALYZE", "parameters": { + "selected_model": 2, "hop_size": 160, "model_elems": 322, "total_frames": 401, @@ -41,6 +42,7 @@ "service": "generic", "message_type": "C7X_MSG_ISTFT_SYNTHESIZE", "parameters": { + "selected_model": 2, "hop_size": 160, "model_elems": 322, "total_frames": 401, diff --git a/example/edge-ai/src/dsp_task_client.cpp b/example/edge-ai/src/dsp_task_client.cpp index 7158222..f094e39 100644 --- a/example/edge-ai/src/dsp_task_client.cpp +++ b/example/edge-ai/src/dsp_task_client.cpp @@ -35,13 +35,14 @@ struct c7x_msg_hdr { int32_t status; }; -struct stft_process_msg { +struct stft_istft_msg { struct c7x_msg_hdr hdr; + uint32_t selected_model; /* ModelId: MODEL_DCCRN=0, MODEL_GTCRN=1, MODEL_GCRN=2, + * MODEL_VGGISH=3, MODEL_YAMNET=4 (see model_config.h) */ uint32_t input_buffer; uint32_t output_buffer; uint32_t input_frame; uint32_t output_frame; - uint32_t graph_id; }; enum c7x_msg_type { @@ -63,7 +64,7 @@ struct deinterleave_interleave_msg { }; static_assert(sizeof(c7x_msg_hdr) == 16); -static_assert(sizeof(stft_process_msg) == 36); +static_assert(sizeof(stft_istft_msg) == 36); static_assert(sizeof(deinterleave_interleave_msg) == 36); enum c7x_status { @@ -156,26 +157,26 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag try { // Determine message type and send appropriate struct if (message_type == "C7X_MSG_STFT_ANALYZE") { - struct stft_process_msg req = {}; + struct stft_istft_msg req = {}; req.hdr.type = C7X_MSG_STFT_ANALYZE; req.hdr.seq = sequence_number_++; - req.hdr.len = sizeof(struct stft_process_msg); + req.hdr.len = sizeof(struct stft_istft_msg); req.hdr.status = 0; + req.selected_model = parameter_value(parameters, "selected_model", 0); req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); req.input_frame = parameter_value(parameters, "input_frame", 0); req.output_frame = parameter_value(parameters, "output_frame", 0); - req.graph_id = parameter_value(parameters, "graph_id", 0); #ifdef DEBUG std::cout << "[GenericClient] STFT_ANALYZE - Sending to firmware:" << std::endl; + std::cout << "[GenericClient] selected_model=" << req.selected_model << std::endl; std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.input_buffer << std::endl; std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.output_buffer << std::endl; std::cout << "[GenericClient] input_frame=" << std::dec << req.input_frame << " frames" << std::endl; std::cout << "[GenericClient] output_frame=" << std::dec << req.output_frame << " frames" << std::endl; - std::cout << "[GenericClient] graph_id=" << req.graph_id << std::endl; #endif - struct stft_process_msg resp = {}; + struct stft_istft_msg resp = {}; if (!exchange_message(rpmsg_fd_, req, resp)) { result.error_message = "STFT analyze message exchange failed"; return result; @@ -201,26 +202,26 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag result.output_size = resp.output_frame; } else if (message_type == "C7X_MSG_ISTFT_SYNTHESIZE") { - struct stft_process_msg req = {}; + struct stft_istft_msg req = {}; req.hdr.type = C7X_MSG_ISTFT_SYNTHESIZE; req.hdr.seq = sequence_number_++; - req.hdr.len = sizeof(struct stft_process_msg); + req.hdr.len = sizeof(struct stft_istft_msg); req.hdr.status = 0; + req.selected_model = parameter_value(parameters, "selected_model", 0); req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); req.input_frame = parameter_value(parameters, "input_frame", 0); req.output_frame = parameter_value(parameters, "output_frame", 0); - req.graph_id = parameter_value(parameters, "graph_id", 0); #ifdef DEBUG std::cout << "[GenericClient] ISTFT_SYNTHESIZE - Sending to firmware:" << std::endl; + std::cout << "[GenericClient] selected_model=" << req.selected_model << std::endl; std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.input_buffer << std::endl; std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.output_buffer << std::endl; std::cout << "[GenericClient] input_frame=" << std::dec << req.input_frame << " frames" << std::endl; std::cout << "[GenericClient] output_frame=" << std::dec << req.output_frame << " frames" << std::endl; - std::cout << "[GenericClient] graph_id=" << req.graph_id << std::endl; #endif - struct stft_process_msg resp = {}; + struct stft_istft_msg resp = {}; if (!exchange_message(rpmsg_fd_, req, resp)) { result.error_message = "ISTFT synthesize message exchange failed"; return result; From f367a5ee923c878fb4c175c771b0419503ec0b14 Mon Sep 17 00:00:00 2001 From: Paresh Bhagat Date: Mon, 24 Aug 2026 17:41:39 +0530 Subject: [PATCH 5/7] example: edge-ai: Add audio classification pipeline for YAMNet/VGGish Add a new pipeline type 'audio_classification' for analysis-only models (YAMNet, VGGish) that run STFT log-mel extraction without ISTFT synthesis. Pipeline details: selected_model=4, hop_size=160, model_elems=64, total_frames=96 Signed-off-by: Paresh Bhagat --- example/edge-ai/CMakeLists.txt | 2 + .../include/audio_classification_pipeline.h | 12 ++ .../pipeline_audio_classification.json | 23 +++ .../src/audio_classification_pipeline.cpp | 181 ++++++++++++++++++ example/edge-ai/src/pipeline_manager.cpp | 3 + 5 files changed, 221 insertions(+) create mode 100644 example/edge-ai/include/audio_classification_pipeline.h create mode 100644 example/edge-ai/json_files/pipeline_audio_classification.json create mode 100644 example/edge-ai/src/audio_classification_pipeline.cpp diff --git a/example/edge-ai/CMakeLists.txt b/example/edge-ai/CMakeLists.txt index b37a168..22735b5 100644 --- a/example/edge-ai/CMakeLists.txt +++ b/example/edge-ai/CMakeLists.txt @@ -34,6 +34,7 @@ set(EDGE_AI_SOURCES src/tvm_pipeline.cpp src/stft_istft_pipeline.cpp src/audio_enhancement_pipeline.cpp + src/audio_classification_pipeline.cpp ) # Create the executable @@ -136,6 +137,7 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_stft_istft.json ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_tvm_inference.json ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_audio_enhancement.json +${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_audio_classification.json DESTINATION ${CMAKE_INSTALL_DATADIR}/tvm_inference/json) # Install input files to /usr/share/tvm_inference/input/ diff --git a/example/edge-ai/include/audio_classification_pipeline.h b/example/edge-ai/include/audio_classification_pipeline.h new file mode 100644 index 0000000..cdc4535 --- /dev/null +++ b/example/edge-ai/include/audio_classification_pipeline.h @@ -0,0 +1,12 @@ +#ifndef AUDIO_CLASSIFICATION_PIPELINE_H +#define AUDIO_CLASSIFICATION_PIPELINE_H + +#include "pipeline_manager.h" +#include "dsp_task_client.h" + +PipelineManager::CommandResult run_audio_classification_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + bool debug); + +#endif // AUDIO_CLASSIFICATION_PIPELINE_H diff --git a/example/edge-ai/json_files/pipeline_audio_classification.json b/example/edge-ai/json_files/pipeline_audio_classification.json new file mode 100644 index 0000000..e619252 --- /dev/null +++ b/example/edge-ai/json_files/pipeline_audio_classification.json @@ -0,0 +1,23 @@ +{ + "pipeline_type": "audio_classification", + "description": "YAMNet: audio -> STFT log-mel spectrogram -> mel_features_output.bin", + "input_file": "/usr/share/tvm_inference/input/input_audio.wav", + "dsp_config": { + "proc_id": 8, + "endpoint": 13 + }, + "stages": [ + { + "stage_id": "stft_analysis", + "service": "generic", + "message_type": "C7X_MSG_STFT_ANALYZE", + "parameters": { + "selected_model": 4, + "hop_size": 160, + "model_elems": 64, + "total_frames": 96, + "batch_n": 64 + } + } + ] +} diff --git a/example/edge-ai/src/audio_classification_pipeline.cpp b/example/edge-ai/src/audio_classification_pipeline.cpp new file mode 100644 index 0000000..01c1103 --- /dev/null +++ b/example/edge-ai/src/audio_classification_pipeline.cpp @@ -0,0 +1,181 @@ +#include "audio_classification_pipeline.h" +#include "pipeline_common.h" +#include "audio_utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +size_t require_param(const std::map& params, + const char* key, const char* stage) +{ + auto it = params.find(key); + if (it == params.end()) + throw PipelineError{std::string{"Stage missing required parameter: "} + key + + " (stage: " + stage + ")"}; + int v = std::stoi(it->second); + if (v <= 0) + throw PipelineError{std::string{"Parameter must be positive: "} + key}; + return static_cast(v); +} + +} // namespace + +PipelineManager::CommandResult run_audio_classification_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + bool debug) +{ + try { + if (state.input_type != PipelineManager::InputType::AUDIO_WAV) + throw PipelineError{"audio_classification pipeline requires a .wav input file"}; + + const PipelineManager::PipelineStage* stft_stage_ptr = nullptr; + for (const auto& stage : state.pipeline_config.stages) { + if (stage.message_type == "C7X_MSG_STFT_ANALYZE") { + stft_stage_ptr = &stage; + break; + } + } + if (!stft_stage_ptr) + throw PipelineError{"audio_classification pipeline requires a C7X_MSG_STFT_ANALYZE stage"}; + + const auto& sp = stft_stage_ptr->parameters; + const size_t HOP_SIZE = require_param(sp, "hop_size", stft_stage_ptr->stage_id.c_str()); + const size_t MODEL_ELEMS = require_param(sp, "model_elems", stft_stage_ptr->stage_id.c_str()); + const size_t TOTAL_FRAMES = require_param(sp, "total_frames", stft_stage_ptr->stage_id.c_str()); + const size_t BATCH_N = require_param(sp, "batch_n", stft_stage_ptr->stage_id.c_str()); + const size_t PAD_FRAMES = TOTAL_FRAMES % BATCH_N; + const size_t NUM_BATCHES = (TOTAL_FRAMES + BATCH_N - 1) / BATCH_N; + + const size_t audio_batch_bytes = BATCH_N * HOP_SIZE * sizeof(int16_t); + const size_t spectral_buf_bytes = TOTAL_FRAMES * MODEL_ELEMS * sizeof(float); + + std::cout << "[App] STFT parameters: HOP=" << HOP_SIZE + << " MODEL_ELEMS=" << MODEL_ELEMS + << " BATCH_N=" << BATCH_N + << " TOTAL_FRAMES=" << TOTAL_FRAMES << std::endl; + + std::vector audio_data; + if (!loadAudioFile(state.current_input_file, audio_data)) + throw PipelineError{"Failed to load input audio"}; + + if (!dsp_client.initialize(state.pipeline_config.dsp_config.proc_id, + state.pipeline_config.dsp_config.endpoint)) + throw PipelineError{"Failed to initialize DSP Task client"}; + + DmaBuffer dma_audio{audio_batch_bytes, "STFT audio input"}; + DmaBuffer dma_mel {spectral_buf_bytes, "STFT mel output"}; + + std::cout << "[App] DMA buffers:" << std::endl; + std::cout << "[App] audio in: phys=0x" << std::hex << dma_audio->phys_addr + << std::dec << " size=" << dma_audio->size << std::endl; + std::cout << "[App] mel out: phys=0x" << std::hex << dma_mel->phys_addr + << std::dec << " size=" << dma_mel->size << std::endl; + + const size_t total_frames = (audio_data.size() + HOP_SIZE - 1) / HOP_SIZE; + audio_data.resize(total_frames * HOP_SIZE, int16_t{0}); + + const size_t num_full_chunks = total_frames / TOTAL_FRAMES; + const size_t partial_frames = total_frames % TOTAL_FRAMES; + const size_t num_chunks = num_full_chunks + (partial_frames > 0 ? 1 : 0); + + std::cout << "[App] Total frames: " << total_frames + << " | Full chunks: " << num_full_chunks + << " | Partial: " << partial_frames << " frames" << std::endl; + + // Accumulate all mel floats from every chunk + std::vector all_mel; + all_mel.reserve(total_frames * MODEL_ELEMS); + + auto t_total = std::chrono::steady_clock::now(); + + for (size_t chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + const size_t chunk_frame_offset = chunk_idx * TOTAL_FRAMES; + const size_t real_frames_chunk = (chunk_idx == num_full_chunks && partial_frames > 0) + ? partial_frames : TOTAL_FRAMES; + auto t_chunk = std::chrono::steady_clock::now(); + + for (size_t batch_idx = 0; batch_idx < NUM_BATCHES; ++batch_idx) { + const size_t frames_this_batch = (batch_idx < NUM_BATCHES - 1) ? BATCH_N : PAD_FRAMES; + const size_t samples_this_batch = frames_this_batch * HOP_SIZE; + const size_t audio_offset = (chunk_frame_offset + batch_idx * BATCH_N) * HOP_SIZE; + const uint64_t spectral_offset = batch_idx * BATCH_N * MODEL_ELEMS * sizeof(float); + + dma_audio.begin_cpu_access(); + std::fill_n(dma_audio.data(), audio_batch_bytes, std::byte{}); + if (audio_offset < audio_data.size()) { + const size_t available = std::min(samples_this_batch, + audio_data.size() - audio_offset); + std::copy_n(audio_data.begin() + static_cast(audio_offset), + available, dma_audio.data()); + } + dma_audio.end_cpu_access(); + + auto params = stft_stage_ptr->parameters; + params["input_buffer"] = hex_address(dma_audio->phys_addr); + params["output_buffer"] = hex_address(dma_mel->phys_addr + spectral_offset); + params["input_frame"] = std::to_string(frames_this_batch); + params["output_frame"] = std::to_string(frames_this_batch); + + if (debug) + std::cout << "[App] batch " << (batch_idx + 1) << "/" << NUM_BATCHES + << ": " << frames_this_batch << " frames" + << " in=0x" << std::hex << dma_audio->phys_addr + << " out=0x" << (dma_mel->phys_addr + spectral_offset) + << std::dec << std::endl; + + auto r = dsp_client.process("C7X_MSG_STFT_ANALYZE", params); + if (!r.success) + throw PipelineError{"STFT batch " + std::to_string(batch_idx + 1) + + " failed: " + r.error_message}; + } + + // Collect real mel frames from this chunk + dma_mel.begin_cpu_access(); + const float* mel_ptr = dma_mel.data(); + all_mel.insert(all_mel.end(), mel_ptr, mel_ptr + real_frames_chunk * MODEL_ELEMS); + dma_mel.end_cpu_access(); + + double t_chunk_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_chunk).count() / 1000.0; + + std::cout << "[App] Chunk " << (chunk_idx + 1) << "/" << num_chunks + << " [" << real_frames_chunk << " frames, " + << real_frames_chunk * MODEL_ELEMS << " mel floats]" + << " " << std::fixed << std::setprecision(1) << t_chunk_ms << "ms" + << std::endl; + } + + double t_total_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t_total).count() / 1000.0; + + std::cout << "[App] Done: " << (all_mel.size() / MODEL_ELEMS) << " mel frames, " + << all_mel.size() << " floats" + << " | " << std::fixed << std::setprecision(1) << t_total_ms << "ms" << std::endl; + + // Save all mel features to binary file + const std::string out_path = "mel_features_output.bin"; + std::ofstream out(out_path, std::ios::binary); + if (!out.is_open()) + throw PipelineError{"Cannot open output file: " + out_path}; + out.write(reinterpret_cast(all_mel.data()), + static_cast(all_mel.size() * sizeof(float))); + std::cout << "[App] Saved to " << out_path + << " (" << (all_mel.size() * sizeof(float)) << " bytes)" << std::endl; + + return PipelineManager::CommandResult::SUCCESS; + + } catch (const std::exception& error) { + std::cerr << "[App] Pipeline failed: " << error.what() << std::endl; + return PipelineManager::CommandResult::ERROR; + } +} diff --git a/example/edge-ai/src/pipeline_manager.cpp b/example/edge-ai/src/pipeline_manager.cpp index 50958a1..83c0478 100644 --- a/example/edge-ai/src/pipeline_manager.cpp +++ b/example/edge-ai/src/pipeline_manager.cpp @@ -4,6 +4,7 @@ #include "tvm_pipeline.h" #include "stft_istft_pipeline.h" #include "audio_enhancement_pipeline.h" +#include "audio_classification_pipeline.h" #include #include #include @@ -164,6 +165,8 @@ int PipelineManager::run_from_json_file(const std::string& json_file_path) result = run_audio_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); } else if (pipeline_type == "stft_istft") { result = run_stft_istft_pipeline(state_, *generic_client_, debug_); + } else if (pipeline_type == "audio_classification") { + result = run_audio_classification_pipeline(state_, *generic_client_, debug_); } else { std::cout << "[App] Error: Unknown pipeline_type: " << pipeline_type << std::endl; return -1; From 795fb20a73c0e82773d932a1db28c94b45650984 Mon Sep 17 00:00:00 2001 From: Paresh Bhagat Date: Wed, 26 Aug 2026 12:51:23 +0530 Subject: [PATCH 6/7] example: edge-ai: Unify DSP message struct and cleanup redundant functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename pipeline types, files and functions audio→speech: - Replace stft_istft_msg and deinterleave_interleave_msg with a single flat dsp_msg (param0..param4) in dsp_task_client.cpp. - Clean up dsp_task_client: remove unused functions. - Rename pipeline types, files and functions from audio to speech. Signed-off-by: Paresh Bhagat --- example/edge-ai/CMakeLists.txt | 21 +- .../include/audio_classification_pipeline.h | 12 - example/edge-ai/include/dsp_task_client.h | 42 +--- .../include/speech_classification_pipeline.h | 12 + ...peline.h => speech_enhancement_pipeline.h} | 8 +- ...on => pipeline_speech_classification.json} | 4 +- ....json => pipeline_speech_enhancement.json} | 4 +- example/edge-ai/src/audio_utils.cpp | 9 +- example/edge-ai/src/dsp_task_client.cpp | 209 ++++++++---------- example/edge-ai/src/main.cpp | 2 +- example/edge-ai/src/pipeline_manager.cpp | 12 +- ...cpp => speech_classification_pipeline.cpp} | 8 +- ...ne.cpp => speech_enhancement_pipeline.cpp} | 6 +- example/edge-ai/src/tvm_pipeline.cpp | 27 --- 14 files changed, 136 insertions(+), 240 deletions(-) delete mode 100644 example/edge-ai/include/audio_classification_pipeline.h create mode 100644 example/edge-ai/include/speech_classification_pipeline.h rename example/edge-ai/include/{audio_enhancement_pipeline.h => speech_enhancement_pipeline.h} (54%) rename example/edge-ai/json_files/{pipeline_audio_classification.json => pipeline_speech_classification.json} (76%) rename example/edge-ai/json_files/{pipeline_audio_enhancement.json => pipeline_speech_enhancement.json} (90%) rename example/edge-ai/src/{audio_classification_pipeline.cpp => speech_classification_pipeline.cpp} (96%) rename example/edge-ai/src/{audio_enhancement_pipeline.cpp => speech_enhancement_pipeline.cpp} (99%) diff --git a/example/edge-ai/CMakeLists.txt b/example/edge-ai/CMakeLists.txt index 22735b5..8dd3673 100644 --- a/example/edge-ai/CMakeLists.txt +++ b/example/edge-ai/CMakeLists.txt @@ -1,5 +1,4 @@ -# TVM Inference Client - uses TVM runtime with existing Python artifacts -# Replicates functionality of inference.py using C++ and TVM runtime +# Edge-AI RPMsg inference example cmake_minimum_required(VERSION 3.10) project(rpmsg_inference_example) @@ -23,7 +22,7 @@ endif() message(STATUS "Using TVM_ROOT: ${TVM_ROOT}") -# Source files (Interactive Pipeline Application) +# Source files set(EDGE_AI_SOURCES src/main.cpp src/tvm_inference_client.cpp @@ -33,8 +32,8 @@ set(EDGE_AI_SOURCES src/audio_utils.cpp src/tvm_pipeline.cpp src/stft_istft_pipeline.cpp - src/audio_enhancement_pipeline.cpp - src/audio_classification_pipeline.cpp + src/speech_enhancement_pipeline.cpp + src/speech_classification_pipeline.cpp ) # Create the executable @@ -80,21 +79,15 @@ endif() # Add json-c include directory target_include_directories(rpmsg_inference_example PRIVATE ${JSON_C_INCLUDE_DIRS}) -# Find readline library for interactive mode with command line editing -find_library(READLINE_LIB readline REQUIRED) - -# Find audio libraries for WAV file processing and playback +# Find audio libraries for WAV file processing find_library(SNDFILE_LIB sndfile REQUIRED) -find_library(ALSA_LIB asound REQUIRED) # Link with required libraries target_link_libraries(rpmsg_inference_example ${TVM_RUNTIME_LIB} ti_rpmsg_dma ${JSON_C_LIBRARIES} - ${READLINE_LIB} ${SNDFILE_LIB} - ${ALSA_LIB} pthread dl ) @@ -136,8 +129,8 @@ install(TARGETS tvm_model_daemon DESTINATION ${CMAKE_INSTALL_BINDIR}) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_stft_istft.json ${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_tvm_inference.json -${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_audio_enhancement.json -${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_audio_classification.json +${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_speech_enhancement.json +${CMAKE_CURRENT_SOURCE_DIR}/json_files/pipeline_speech_classification.json DESTINATION ${CMAKE_INSTALL_DATADIR}/tvm_inference/json) # Install input files to /usr/share/tvm_inference/input/ diff --git a/example/edge-ai/include/audio_classification_pipeline.h b/example/edge-ai/include/audio_classification_pipeline.h deleted file mode 100644 index cdc4535..0000000 --- a/example/edge-ai/include/audio_classification_pipeline.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef AUDIO_CLASSIFICATION_PIPELINE_H -#define AUDIO_CLASSIFICATION_PIPELINE_H - -#include "pipeline_manager.h" -#include "dsp_task_client.h" - -PipelineManager::CommandResult run_audio_classification_pipeline( - PipelineManager::State& state, - DspTaskClient& dsp_client, - bool debug); - -#endif // AUDIO_CLASSIFICATION_PIPELINE_H diff --git a/example/edge-ai/include/dsp_task_client.h b/example/edge-ai/include/dsp_task_client.h index 8f976f7..eef4498 100644 --- a/example/edge-ai/include/dsp_task_client.h +++ b/example/edge-ai/include/dsp_task_client.h @@ -11,11 +11,8 @@ extern "C" { } /** - * @brief Generic Task client for communicating with DSP Generic Service - * - * Handles communication with the Generic Service running on DSP endpoint 13. - * Message type determines which struct and processing logic to use. - * Uses zero-copy approach with TVM shared memory regions. + * @brief Client for communicating with the DSP generic service over RPMsg. + * DSP endpoint and proc_id are configured at initialize() time via JSON pipeline config. */ class DspTaskClient { public: @@ -35,42 +32,12 @@ class DspTaskClient { * @param max_output_size Maximum output buffer size in bytes * @return true on success, false on failure */ - bool initialize(int proc_id, int endpoint, - uint32_t max_input_size = 1024*1024, - uint32_t max_output_size = 1024*1024); - - /** - * @brief Generic processing function - * @param message_type Message type string (determines struct and processing) - * @param input_data Input data buffer (not used in zero-copy mode) - * @param input_size Size of input data in bytes - * @param output_data Output data buffer (not used in zero-copy mode) - * @param output_size Size of output buffer in bytes - * @return ProcessingResult with status and information - */ - ProcessingResult process(const std::string& message_type, - void* input_data, - uint32_t input_size, - void* output_data, - uint32_t output_size, - const std::map& parameters = {}); + bool initialize(int proc_id, int endpoint); ProcessingResult process( const std::string& message_type, const std::map& parameters = {}); - /** - * @brief Get status from the Generic Service - * @return ProcessingResult with service statistics - */ - ProcessingResult get_service_status(); - - /** - * @brief Ping the Generic Service - * @return true if service responds, false otherwise - */ - bool ping_service(); - /** * @brief Shutdown and cleanup */ @@ -93,9 +60,6 @@ class DspTaskClient { // Internal methods bool open_rpmsg_device(); void close_rpmsg_device(); - bool allocate_shared_buffers(); - void free_shared_buffers(); - std::string get_error_string(int32_t error_code); }; #endif // DSP_TASK_CLIENT_H diff --git a/example/edge-ai/include/speech_classification_pipeline.h b/example/edge-ai/include/speech_classification_pipeline.h new file mode 100644 index 0000000..d253e00 --- /dev/null +++ b/example/edge-ai/include/speech_classification_pipeline.h @@ -0,0 +1,12 @@ +#ifndef SPEECH_CLASSIFICATION_PIPELINE_H +#define SPEECH_CLASSIFICATION_PIPELINE_H + +#include "pipeline_manager.h" +#include "dsp_task_client.h" + +PipelineManager::CommandResult run_speech_classification_pipeline( + PipelineManager::State& state, + DspTaskClient& dsp_client, + bool debug); + +#endif // SPEECH_CLASSIFICATION_PIPELINE_H diff --git a/example/edge-ai/include/audio_enhancement_pipeline.h b/example/edge-ai/include/speech_enhancement_pipeline.h similarity index 54% rename from example/edge-ai/include/audio_enhancement_pipeline.h rename to example/edge-ai/include/speech_enhancement_pipeline.h index c00d8c6..7290cc0 100644 --- a/example/edge-ai/include/audio_enhancement_pipeline.h +++ b/example/edge-ai/include/speech_enhancement_pipeline.h @@ -1,14 +1,14 @@ -#ifndef AUDIO_ENHANCEMENT_PIPELINE_H -#define AUDIO_ENHANCEMENT_PIPELINE_H +#ifndef SPEECH_ENHANCEMENT_PIPELINE_H +#define SPEECH_ENHANCEMENT_PIPELINE_H #include "pipeline_manager.h" #include "dsp_task_client.h" #include "tvm_inference_client.h" -PipelineManager::CommandResult run_audio_enhancement_pipeline( +PipelineManager::CommandResult run_speech_enhancement_pipeline( PipelineManager::State& state, DspTaskClient& dsp_client, TvmInferenceClient& tvm_client, bool debug); -#endif // AUDIO_ENHANCEMENT_PIPELINE_H +#endif // SPEECH_ENHANCEMENT_PIPELINE_H diff --git a/example/edge-ai/json_files/pipeline_audio_classification.json b/example/edge-ai/json_files/pipeline_speech_classification.json similarity index 76% rename from example/edge-ai/json_files/pipeline_audio_classification.json rename to example/edge-ai/json_files/pipeline_speech_classification.json index e619252..8915deb 100644 --- a/example/edge-ai/json_files/pipeline_audio_classification.json +++ b/example/edge-ai/json_files/pipeline_speech_classification.json @@ -1,6 +1,6 @@ { - "pipeline_type": "audio_classification", - "description": "YAMNet: audio -> STFT log-mel spectrogram -> mel_features_output.bin", + "pipeline_type": "speech_classification", + "description": "YAMNet: speech -> STFT log-mel spectrogram -> mel_features_output.bin", "input_file": "/usr/share/tvm_inference/input/input_audio.wav", "dsp_config": { "proc_id": 8, diff --git a/example/edge-ai/json_files/pipeline_audio_enhancement.json b/example/edge-ai/json_files/pipeline_speech_enhancement.json similarity index 90% rename from example/edge-ai/json_files/pipeline_audio_enhancement.json rename to example/edge-ai/json_files/pipeline_speech_enhancement.json index 608ed52..d162909 100644 --- a/example/edge-ai/json_files/pipeline_audio_enhancement.json +++ b/example/edge-ai/json_files/pipeline_speech_enhancement.json @@ -1,6 +1,6 @@ { - "pipeline_type": "audio_enhancement", - "description": "GCRN 3-stage audio pipeline: STFT analyze -> TVM inference -> ISTFT synthesize", + "pipeline_type": "speech_enhancement", + "description": "GCRN 3-stage speech pipeline: STFT analyze -> TVM inference -> ISTFT synthesize", "input_file": "/usr/share/tvm_inference/input/input_audio.wav", "artifacts_path": "/usr/share/tvm_inference/artifacts/", "dsp_config": { diff --git a/example/edge-ai/src/audio_utils.cpp b/example/edge-ai/src/audio_utils.cpp index 621dafc..991878c 100644 --- a/example/edge-ai/src/audio_utils.cpp +++ b/example/edge-ai/src/audio_utils.cpp @@ -31,8 +31,7 @@ bool loadAudioFile(const std::string& filename, std::vector& audio_data return false; } - std::cout << "[App] Audio file info: " << (sfinfo.frames / 160) << " GCRN frames (" - << sfinfo.frames << " samples), " + std::cout << "[App] Audio file info: " << sfinfo.frames << " samples, " << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; // Read all audio data @@ -78,8 +77,7 @@ bool saveAudioFile(const std::string& filename, const std::vector& audi } std::cout << "[App] Saving audio to: " << filename << std::endl; - std::cout << "[App] Output file info: " << (audio_data.size() / 160) << " GCRN frames (" - << audio_data.size() << " samples), " + std::cout << "[App] Output file info: " << audio_data.size() << " samples, " << sfinfo.samplerate << "Hz, " << sfinfo.channels << " channel(s)" << std::endl; // Write audio data to file @@ -91,8 +89,7 @@ bool saveAudioFile(const std::string& filename, const std::vector& audi return false; } - std::cout << "[App] Successfully saved " << (frames_written / 160) << " GCRN frames (" - << frames_written << " samples) to " << filename << std::endl; + std::cout << "[App] Successfully saved " << frames_written << " samples to " << filename << std::endl; std::cout << "[App] Duration: " << (static_cast(frames_written) / sfinfo.samplerate) << " seconds" << std::endl; diff --git a/example/edge-ai/src/dsp_task_client.cpp b/example/edge-ai/src/dsp_task_client.cpp index f094e39..f0f2a40 100644 --- a/example/edge-ai/src/dsp_task_client.cpp +++ b/example/edge-ai/src/dsp_task_client.cpp @@ -35,37 +35,49 @@ struct c7x_msg_hdr { int32_t status; }; -struct stft_istft_msg { +/* + * Generic DSP message — flat payload, 5 x uint32_t after the header. + * Wire size is always fixed: sizeof(c7x_msg_hdr) + 5 * sizeof(uint32_t). + * + * Field mapping by message type: + * + * param | C7X_MSG_STFT_ANALYZE | C7X_MSG_ISTFT_SYNTHESIZE | C7X_DEINTERLEAVE_MSG_ANALYZE + * -------|---------------------------|---------------------------|----------------------------- + * param0 | selected_model | selected_model | input_buffer (phys addr) + * param1 | input_buffer (phys addr) | input_buffer (phys addr) | output_buffer (phys addr) + * param2 | output_buffer (phys addr) | output_buffer (phys addr) | input_frame + * param3 | input_frame | input_frame | fft_size + * param4 | output_frame | output_frame | flag (0=deinterleave, 1=interleave) + * + * Note: param0 maps differently per message type because the firmware + * STFT/ISTFT and utils structs have different layouts (see table above). + * + * To add a new message type: + * 1. Add its opcodes to c7x_msg_type below + * 2. Add a column to this table + * 3. Add a new else-if branch in DspTaskClient::process() + * 4. Set unused params to 0 + */ +struct dsp_msg { struct c7x_msg_hdr hdr; - uint32_t selected_model; /* ModelId: MODEL_DCCRN=0, MODEL_GTCRN=1, MODEL_GCRN=2, - * MODEL_VGGISH=3, MODEL_YAMNET=4 (see model_config.h) */ - uint32_t input_buffer; - uint32_t output_buffer; - uint32_t input_frame; - uint32_t output_frame; + uint32_t param0; + uint32_t param1; + uint32_t param2; + uint32_t param3; + uint32_t param4; }; enum c7x_msg_type { - C7X_MSG_STFT_ANALYZE = 0x1020, - C7X_MSG_STFT_ANALYZE_RESP = 0x2020, - C7X_MSG_ISTFT_SYNTHESIZE = 0x1030, - C7X_MSG_ISTFT_SYNTHESIZE_RESP = 0x2030, - C7X_DEINTERLEAVE_MSG_ANALYZE = 0x1040, + C7X_MSG_STFT_ANALYZE = 0x1020, + C7X_MSG_STFT_ANALYZE_RESP = 0x2020, + C7X_MSG_ISTFT_SYNTHESIZE = 0x1030, + C7X_MSG_ISTFT_SYNTHESIZE_RESP = 0x2030, + C7X_DEINTERLEAVE_MSG_ANALYZE = 0x1040, C7X_DEINTERLEAVE_MSG_ANALYZE_RESP = 0x2040 }; -struct deinterleave_interleave_msg { - struct c7x_msg_hdr hdr; - uint32_t input_buffer; - uint32_t output_buffer; - uint32_t input_frame; - uint32_t fft_size; - uint32_t flag; // 0: deinterleave, 1: interleave -}; - static_assert(sizeof(c7x_msg_hdr) == 16); -static_assert(sizeof(stft_istft_msg) == 36); -static_assert(sizeof(deinterleave_interleave_msg) == 36); +static_assert(sizeof(dsp_msg) == sizeof(c7x_msg_hdr) + 5 * sizeof(uint32_t)); enum c7x_status { C7X_STATUS_SUCCESS = 0, @@ -97,11 +109,8 @@ DspTaskClient::~DspTaskClient() shutdown(); } -bool DspTaskClient::initialize(int proc_id, int endpoint, - uint32_t max_input_size, uint32_t max_output_size) +bool DspTaskClient::initialize(int proc_id, int endpoint) { - (void)max_input_size; - (void)max_output_size; if (initialized_) { return true; } @@ -134,18 +143,10 @@ void DspTaskClient::close_rpmsg_device() } } -DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& message_type, - void* input_data, - uint32_t input_size, - void* output_data, - uint32_t output_size, - const std::map& parameters) +DspTaskClient::ProcessingResult DspTaskClient::process( + const std::string& message_type, + const std::map& parameters) { - (void)input_data; - (void)input_size; - (void)output_data; - (void)output_size; - ProcessingResult result = {}; result.success = false; @@ -157,26 +158,26 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag try { // Determine message type and send appropriate struct if (message_type == "C7X_MSG_STFT_ANALYZE") { - struct stft_istft_msg req = {}; - req.hdr.type = C7X_MSG_STFT_ANALYZE; - req.hdr.seq = sequence_number_++; - req.hdr.len = sizeof(struct stft_istft_msg); + struct dsp_msg req = {}; + req.hdr.type = C7X_MSG_STFT_ANALYZE; + req.hdr.seq = sequence_number_++; + req.hdr.len = sizeof(struct dsp_msg); req.hdr.status = 0; - req.selected_model = parameter_value(parameters, "selected_model", 0); - req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); - req.input_frame = parameter_value(parameters, "input_frame", 0); - req.output_frame = parameter_value(parameters, "output_frame", 0); + req.param0 = parameter_value(parameters, "selected_model", 0); /* selected_model */ + req.param1 = parameter_value(parameters, "input_buffer", 0, 16); /* input_buffer */ + req.param2 = parameter_value(parameters, "output_buffer", 0, 16); /* output_buffer */ + req.param3 = parameter_value(parameters, "input_frame", 0); /* input_frame */ + req.param4 = parameter_value(parameters, "output_frame", 0); /* output_frame */ #ifdef DEBUG std::cout << "[GenericClient] STFT_ANALYZE - Sending to firmware:" << std::endl; - std::cout << "[GenericClient] selected_model=" << req.selected_model << std::endl; - std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.input_buffer << std::endl; - std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.output_buffer << std::endl; - std::cout << "[GenericClient] input_frame=" << std::dec << req.input_frame << " frames" << std::endl; - std::cout << "[GenericClient] output_frame=" << std::dec << req.output_frame << " frames" << std::endl; + std::cout << "[GenericClient] selected_model=" << req.param0 << std::endl; + std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.param1 << std::endl; + std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.param2 << std::endl; + std::cout << "[GenericClient] input_frame=" << std::dec << req.param3 << " frames" << std::endl; + std::cout << "[GenericClient] output_frame=" << std::dec << req.param4 << " frames" << std::endl; #endif - struct stft_istft_msg resp = {}; + struct dsp_msg resp = {}; if (!exchange_message(rpmsg_fd_, req, resp)) { result.error_message = "STFT analyze message exchange failed"; return result; @@ -188,9 +189,9 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag } #ifdef DEBUG std::cout << "[GenericClient] STFT_ANALYZE - Firmware responded:" << std::endl; - std::cout << "[GenericClient] status=" << resp.hdr.status << std::endl; - std::cout << "[GenericClient] resp.input_frame=" << resp.input_frame << " frames" << std::endl; - std::cout << "[GenericClient] resp.output_frame=" << resp.output_frame << " frames" << std::endl; + std::cout << "[GenericClient] status=" << resp.hdr.status << std::endl; + std::cout << "[GenericClient] input_frame=" << resp.param3 << " frames" << std::endl; + std::cout << "[GenericClient] output_frame=" << resp.param4 << " frames" << std::endl; #endif if (resp.hdr.status != C7X_STATUS_SUCCESS) { result.error_message = "DSP STFT analyze failed"; @@ -198,30 +199,30 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag } result.success = true; - result.input_size = resp.input_frame; - result.output_size = resp.output_frame; + result.input_size = resp.param3; /* input_frame */ + result.output_size = resp.param4; /* output_frame */ } else if (message_type == "C7X_MSG_ISTFT_SYNTHESIZE") { - struct stft_istft_msg req = {}; - req.hdr.type = C7X_MSG_ISTFT_SYNTHESIZE; - req.hdr.seq = sequence_number_++; - req.hdr.len = sizeof(struct stft_istft_msg); + struct dsp_msg req = {}; + req.hdr.type = C7X_MSG_ISTFT_SYNTHESIZE; + req.hdr.seq = sequence_number_++; + req.hdr.len = sizeof(struct dsp_msg); req.hdr.status = 0; - req.selected_model = parameter_value(parameters, "selected_model", 0); - req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); - req.input_frame = parameter_value(parameters, "input_frame", 0); - req.output_frame = parameter_value(parameters, "output_frame", 0); + req.param0 = parameter_value(parameters, "selected_model", 0); /* selected_model */ + req.param1 = parameter_value(parameters, "input_buffer", 0, 16); /* input_buffer */ + req.param2 = parameter_value(parameters, "output_buffer", 0, 16); /* output_buffer */ + req.param3 = parameter_value(parameters, "input_frame", 0); /* input_frame */ + req.param4 = parameter_value(parameters, "output_frame", 0); /* output_frame */ #ifdef DEBUG std::cout << "[GenericClient] ISTFT_SYNTHESIZE - Sending to firmware:" << std::endl; - std::cout << "[GenericClient] selected_model=" << req.selected_model << std::endl; - std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.input_buffer << std::endl; - std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.output_buffer << std::endl; - std::cout << "[GenericClient] input_frame=" << std::dec << req.input_frame << " frames" << std::endl; - std::cout << "[GenericClient] output_frame=" << std::dec << req.output_frame << " frames" << std::endl; + std::cout << "[GenericClient] selected_model=" << req.param0 << std::endl; + std::cout << "[GenericClient] input_buffer=0x" << std::hex << req.param1 << std::endl; + std::cout << "[GenericClient] output_buffer=0x" << std::hex << req.param2 << std::endl; + std::cout << "[GenericClient] input_frame=" << std::dec << req.param3 << " frames" << std::endl; + std::cout << "[GenericClient] output_frame=" << std::dec << req.param4 << " frames" << std::endl; #endif - struct stft_istft_msg resp = {}; + struct dsp_msg resp = {}; if (!exchange_message(rpmsg_fd_, req, resp)) { result.error_message = "ISTFT synthesize message exchange failed"; return result; @@ -233,9 +234,9 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag } #ifdef DEBUG std::cout << "[GenericClient] ISTFT_SYNTHESIZE - Firmware responded:" << std::endl; - std::cout << "[GenericClient] status=" << resp.hdr.status << std::endl; - std::cout << "[GenericClient] resp.input_frame=" << resp.input_frame << " frames" << std::endl; - std::cout << "[GenericClient] resp.output_frame=" << resp.output_frame << " frames" << std::endl; + std::cout << "[GenericClient] status=" << resp.hdr.status << std::endl; + std::cout << "[GenericClient] input_frame=" << resp.param3 << " frames" << std::endl; + std::cout << "[GenericClient] output_frame=" << resp.param4 << " frames" << std::endl; #endif if (resp.hdr.status != C7X_STATUS_SUCCESS) { result.error_message = "DSP ISTFT synthesize failed"; @@ -243,23 +244,23 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag } result.success = true; - result.input_size = resp.input_frame; - result.output_size = resp.output_frame; + result.input_size = resp.param3; /* input_frame */ + result.output_size = resp.param4; /* output_frame */ } else if (message_type == "C7X_DEINTERLEAVE_MSG_ANALYZE") { - struct deinterleave_interleave_msg req = {}; + struct dsp_msg req = {}; req.hdr.type = C7X_DEINTERLEAVE_MSG_ANALYZE; req.hdr.seq = sequence_number_++; - req.hdr.len = sizeof(struct deinterleave_interleave_msg); + req.hdr.len = sizeof(struct dsp_msg); req.hdr.status = 0; - req.input_buffer = parameter_value(parameters, "input_buffer", 0, 16); - req.output_buffer = parameter_value(parameters, "output_buffer", 0, 16); - req.input_frame = parameter_value(parameters, "input_frame", 0); - req.fft_size = parameter_value(parameters, "fft_size", 0); - req.flag = parameter_value(parameters, "flag", 0); + req.param0 = parameter_value(parameters, "input_buffer", 0, 16); /* input_buffer */ + req.param1 = parameter_value(parameters, "output_buffer", 0, 16); /* output_buffer */ + req.param2 = parameter_value(parameters, "input_frame", 0); /* input_frame */ + req.param3 = parameter_value(parameters, "fft_size", 0); /* fft_size */ + req.param4 = parameter_value(parameters, "flag", 0); /* flag */ - struct deinterleave_interleave_msg resp = {}; + struct dsp_msg resp = {}; if (!exchange_message(rpmsg_fd_, req, resp)) { result.error_message = "Layout conversion message exchange failed"; return result; @@ -276,8 +277,8 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag } result.success = true; - result.input_size = resp.input_frame; - result.output_size = resp.input_frame; + result.input_size = resp.param2; /* input_frame */ + result.output_size = resp.param2; /* input_frame */ } else { result.error_message = "Unknown message type: " + message_type; @@ -291,29 +292,6 @@ DspTaskClient::ProcessingResult DspTaskClient::process(const std::string& messag return result; } -DspTaskClient::ProcessingResult DspTaskClient::process( - const std::string& message_type, - const std::map& parameters) -{ - return process(message_type, nullptr, 0, nullptr, 0, parameters); -} - -DspTaskClient::ProcessingResult DspTaskClient::get_service_status() -{ - ProcessingResult result = {}; - result.success = initialized_; - if (!initialized_) { - result.error_message = "Client not initialized"; - } - return result; -} - -bool DspTaskClient::ping_service() -{ - // STFT service doesn't have separate ping - just return initialized status - return initialized_; -} - void DspTaskClient::shutdown() { if (initialized_) { @@ -321,12 +299,3 @@ void DspTaskClient::shutdown() initialized_ = false; } } - -std::string DspTaskClient::get_error_string(int32_t error_code) -{ - switch (error_code) { - case C7X_STATUS_SUCCESS: return "Success"; - case C7X_STATUS_ERROR: return "Error"; - default: return "Unknown error"; - } -} diff --git a/example/edge-ai/src/main.cpp b/example/edge-ai/src/main.cpp index 5b38b19..8059851 100644 --- a/example/edge-ai/src/main.cpp +++ b/example/edge-ai/src/main.cpp @@ -40,7 +40,7 @@ void print_usage(std::string_view program) << " " << program << " --help Show this help\n\n" << "Examples:\n" << " " << program << " pipeline_tvm_inference.json\n" - << " " << program << " pipeline_audio_enhancement.json --debug\n"; + << " " << program << " pipeline_speech_enhancement.json --debug\n"; } } // namespace diff --git a/example/edge-ai/src/pipeline_manager.cpp b/example/edge-ai/src/pipeline_manager.cpp index 83c0478..8ba0028 100644 --- a/example/edge-ai/src/pipeline_manager.cpp +++ b/example/edge-ai/src/pipeline_manager.cpp @@ -3,8 +3,8 @@ #include "audio_utils.h" #include "tvm_pipeline.h" #include "stft_istft_pipeline.h" -#include "audio_enhancement_pipeline.h" -#include "audio_classification_pipeline.h" +#include "speech_enhancement_pipeline.h" +#include "speech_classification_pipeline.h" #include #include #include @@ -161,12 +161,12 @@ int PipelineManager::run_from_json_file(const std::string& json_file_path) if (pipeline_type == "tvm_only") { result = run_tvm_pipeline(state_, *tvm_client_); - } else if (pipeline_type == "audio_enhancement") { - result = run_audio_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); + } else if (pipeline_type == "speech_enhancement") { + result = run_speech_enhancement_pipeline(state_, *generic_client_, *tvm_client_, debug_); } else if (pipeline_type == "stft_istft") { result = run_stft_istft_pipeline(state_, *generic_client_, debug_); - } else if (pipeline_type == "audio_classification") { - result = run_audio_classification_pipeline(state_, *generic_client_, debug_); + } else if (pipeline_type == "speech_classification") { + result = run_speech_classification_pipeline(state_, *generic_client_, debug_); } else { std::cout << "[App] Error: Unknown pipeline_type: " << pipeline_type << std::endl; return -1; diff --git a/example/edge-ai/src/audio_classification_pipeline.cpp b/example/edge-ai/src/speech_classification_pipeline.cpp similarity index 96% rename from example/edge-ai/src/audio_classification_pipeline.cpp rename to example/edge-ai/src/speech_classification_pipeline.cpp index 01c1103..6f7f30b 100644 --- a/example/edge-ai/src/audio_classification_pipeline.cpp +++ b/example/edge-ai/src/speech_classification_pipeline.cpp @@ -1,4 +1,4 @@ -#include "audio_classification_pipeline.h" +#include "speech_classification_pipeline.h" #include "pipeline_common.h" #include "audio_utils.h" #include @@ -29,14 +29,14 @@ size_t require_param(const std::map& params, } // namespace -PipelineManager::CommandResult run_audio_classification_pipeline( +PipelineManager::CommandResult run_speech_classification_pipeline( PipelineManager::State& state, DspTaskClient& dsp_client, bool debug) { try { if (state.input_type != PipelineManager::InputType::AUDIO_WAV) - throw PipelineError{"audio_classification pipeline requires a .wav input file"}; + throw PipelineError{"speech_classification pipeline requires a .wav input file"}; const PipelineManager::PipelineStage* stft_stage_ptr = nullptr; for (const auto& stage : state.pipeline_config.stages) { @@ -46,7 +46,7 @@ PipelineManager::CommandResult run_audio_classification_pipeline( } } if (!stft_stage_ptr) - throw PipelineError{"audio_classification pipeline requires a C7X_MSG_STFT_ANALYZE stage"}; + throw PipelineError{"speech_classification pipeline requires a C7X_MSG_STFT_ANALYZE stage"}; const auto& sp = stft_stage_ptr->parameters; const size_t HOP_SIZE = require_param(sp, "hop_size", stft_stage_ptr->stage_id.c_str()); diff --git a/example/edge-ai/src/audio_enhancement_pipeline.cpp b/example/edge-ai/src/speech_enhancement_pipeline.cpp similarity index 99% rename from example/edge-ai/src/audio_enhancement_pipeline.cpp rename to example/edge-ai/src/speech_enhancement_pipeline.cpp index 7550673..9689805 100644 --- a/example/edge-ai/src/audio_enhancement_pipeline.cpp +++ b/example/edge-ai/src/speech_enhancement_pipeline.cpp @@ -1,4 +1,4 @@ -#include "audio_enhancement_pipeline.h" +#include "speech_enhancement_pipeline.h" #include "pipeline_common.h" #include "audio_utils.h" #include @@ -30,7 +30,7 @@ size_t require_param(const std::map& params, } // namespace -PipelineManager::CommandResult run_audio_enhancement_pipeline( +PipelineManager::CommandResult run_speech_enhancement_pipeline( PipelineManager::State& state, DspTaskClient& dsp_client, TvmInferenceClient& tvm_client, @@ -61,7 +61,7 @@ PipelineManager::CommandResult run_audio_enhancement_pipeline( } if (!stft_stage_ptr || !istft_stage_ptr || !deint_stage_ptr || !inter_stage_ptr) - throw PipelineError{"audio_enhancement pipeline requires STFT, deinterleave, interleave and ISTFT stages"}; + throw PipelineError{"speech_enhancement pipeline requires STFT, deinterleave, interleave and ISTFT stages"}; // Read loop/buffer parameters from STFT stage (drives STFT-side buffers and loop) const auto& sp = stft_stage_ptr->parameters; diff --git a/example/edge-ai/src/tvm_pipeline.cpp b/example/edge-ai/src/tvm_pipeline.cpp index 9d2f0df..42fe608 100644 --- a/example/edge-ai/src/tvm_pipeline.cpp +++ b/example/edge-ai/src/tvm_pipeline.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace { @@ -31,32 +30,6 @@ bool saveTensorFile(const std::string& filename, const std::vector& tenso return true; } -bool loadBinTensor(const std::string& filename, std::vector& tensor_data) -{ - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if (!file.is_open()) { - std::cout << "[App] Error: Cannot open BIN file: " << filename << std::endl; - return false; - } - - const std::streamsize size = file.tellg(); - if (size <= 0 || size % static_cast(sizeof(float)) != 0) { - std::cout << "[App] Error: BIN file is not a non-empty float32 tensor" << std::endl; - return false; - } - file.seekg(0, std::ios::beg); - - const size_t num_floats = static_cast(size) / sizeof(float); - tensor_data.resize(num_floats); - - if (!file.read(reinterpret_cast(tensor_data.data()), size)) { - std::cout << "[App] Error: Failed to read BIN file" << std::endl; - return false; - } - - return true; -} - } // namespace PipelineManager::CommandResult run_tvm_pipeline( From bba8c8ec50f09fece031b1b750b80ee8bc8108ce Mon Sep 17 00:00:00 2001 From: Vishnu Singh Date: Wed, 26 Aug 2026 13:16:50 +0530 Subject: [PATCH 7/7] am62dxx: speech-enhancement: fix switch_firmware failure from GUI stop - add wait_for_c7x(): checks /lib/firmware/am62d-c71_0-fw symlink target; if wrong, stops C7x, fixes the symlink, starts C7x; then polls /sys/class/remoteproc/remoteproc0/state up to 60s until running; called in main() before model initialization so both boot and service restarts block until C7x is confirmed running with correct firmware Signed-off-by: Vishnu Singh Signed-off-by: Paresh Bhagat --- example/audio_offload/src/metrics.c | 5 +- .../src/speech_enhancement_pipeline.cpp | 6 +- example/edge-ai/src/tvm_model_daemon.cpp | 102 +++++++++++++++++- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/example/audio_offload/src/metrics.c b/example/audio_offload/src/metrics.c index 30b627a..6a0689d 100644 --- a/example/audio_offload/src/metrics.c +++ b/example/audio_offload/src/metrics.c @@ -14,7 +14,10 @@ float get_cpu_load() long user, nice, system, idle; FILE *fp = fopen("/proc/stat", "r"); if (!fp) return -1; - fscanf(fp, "cpu %ld %ld %ld %ld", &user, &nice, &system, &idle); + if (fscanf(fp, "cpu %ld %ld %ld %ld", &user, &nice, &system, &idle) != 4) { + fclose(fp); + return -1; + } fclose(fp); long total = (user-last_user)+(nice-last_nice)+(system-last_system); long total_all = total + (idle-last_idle); diff --git a/example/edge-ai/src/speech_enhancement_pipeline.cpp b/example/edge-ai/src/speech_enhancement_pipeline.cpp index 9689805..0ff1864 100644 --- a/example/edge-ai/src/speech_enhancement_pipeline.cpp +++ b/example/edge-ai/src/speech_enhancement_pipeline.cpp @@ -216,6 +216,7 @@ PipelineManager::CommandResult run_speech_enhancement_pipeline( dma_buf1.begin_cpu_access(); std::fill_n(dma_buf1.data(), audio_batch_bytes, std::byte{}); + audio_stream.send_frame(0, dma_buf1.data(), audio_bytes); if (audio_offset < audio_data.size()) { const size_t available = std::min(samples_this_batch, audio_data.size() - audio_offset); @@ -223,7 +224,6 @@ PipelineManager::CommandResult run_speech_enhancement_pipeline( available, dma_buf1.data()); } dma_buf1.end_cpu_access(); - audio_stream.send_frame(0, dma_buf1.data(), audio_bytes); auto params = stft_stage_ptr->parameters; params["input_buffer"] = hex_address(dma_buf1->phys_addr); @@ -363,6 +363,7 @@ PipelineManager::CommandResult run_speech_enhancement_pipeline( } std::copy_n(out_ptr, samples_this_batch, std::back_inserter(chunk_output)); + audio_stream.send_frame(1, out_ptr, samples_this_batch * sizeof(int16_t)); dma_buf4.end_cpu_access(); } double t_istft_ms = std::chrono::duration_cast( @@ -374,9 +375,6 @@ PipelineManager::CommandResult run_speech_enhancement_pipeline( std::copy(chunk_output.begin() + static_cast(lo_sample), chunk_output.begin() + static_cast(keep_end), std::back_inserter(processed_audio_data)); - audio_stream.send_frame(1, - chunk_output.data() + lo_sample, - (keep_end - lo_sample) * sizeof(int16_t)); } double t_chunk_ms = std::chrono::duration_cast( diff --git a/example/edge-ai/src/tvm_model_daemon.cpp b/example/edge-ai/src/tvm_model_daemon.cpp index 6c0d445..e700b61 100644 --- a/example/edge-ai/src/tvm_model_daemon.cpp +++ b/example/edge-ai/src/tvm_model_daemon.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -28,6 +30,99 @@ static constexpr const char* DEFAULT_ARTIFACTS = "/usr/share/tvm_inference/artifacts/"; +static constexpr const char* C7X_FW_LINK = "/lib/firmware/am62d-c71_0-fw"; +static constexpr const char* C7X_FW_TARGET = "/lib/firmware/ti-ipc/am62dxx/dsp_edgeai.c75ss0-0.release.strip.out"; +static constexpr const char* RPROC_STATE = "/sys/class/remoteproc/remoteproc0/state"; +static constexpr int BOOT_TIMEOUT_S = 60; +static constexpr int STOP_TIMEOUT_S = 1; + +static std::string read_sysfs(const char* path) { + int fd = ::open(path, O_RDONLY); + if (fd < 0) return ""; + char buf[64] = {}; + ssize_t n = ::read(fd, buf, sizeof(buf) - 1); + ::close(fd); + if (n <= 0) return ""; + if (n > 0 && buf[n - 1] == '\n') buf[n - 1] = '\0'; + return std::string(buf); +} + +static bool write_sysfs(const char* path, const char* value) { + int fd = ::open(path, O_WRONLY); + if (fd < 0) return false; + ssize_t n = ::write(fd, value, std::strlen(value)); + ::close(fd); + return n > 0; +} + +static bool fw_link_correct() { + char target[512] = {}; + ssize_t n = ::readlink(C7X_FW_LINK, target, sizeof(target) - 1); + if (n < 0) return false; + target[n] = '\0'; + return std::strcmp(target, C7X_FW_TARGET) == 0; +} + +/* Ensure C7x firmware symlink points to the edgeai binary and C7x is running. + * If the symlink is wrong: stop C7x, fix symlink, start C7x. + * If the symlink is correct but C7x is not running: start C7x. + * Polls until running state or BOOT_TIMEOUT_S seconds. */ +static bool wait_for_c7x() { + bool need_restart = false; + + if (!fw_link_correct()) { + std::cout << "[daemon] C7x firmware symlink incorrect — fixing\n"; + + std::string state = read_sysfs(RPROC_STATE); + if (state == "running" || state == "attached") { + std::cout << "[daemon] Stopping C7x for firmware update...\n"; + if (!write_sysfs(RPROC_STATE, "stop")) { + std::cerr << "[daemon] Failed to stop C7x: " << std::strerror(errno) << '\n'; + return false; + } + for (int i = 0; i < STOP_TIMEOUT_S; ++i) { + ::sleep(1); + if (read_sysfs(RPROC_STATE) == "offline") break; + } + } + + ::unlink(C7X_FW_LINK); + if (::symlink(C7X_FW_TARGET, C7X_FW_LINK) != 0) { + std::cerr << "[daemon] Failed to create firmware symlink: " << std::strerror(errno) << '\n'; + return false; + } + std::cout << "[daemon] Firmware symlink updated -> " << C7X_FW_TARGET << '\n'; + need_restart = true; + } + + std::string state = read_sysfs(RPROC_STATE); + if (state != "running") { + std::cout << "[daemon] Starting C7x...\n"; + if (!write_sysfs(RPROC_STATE, "start")) { + std::cerr << "[daemon] Failed to start C7x: " << std::strerror(errno) << '\n'; + return false; + } + need_restart = true; + } + + if (!need_restart && state == "running") { + std::cout << "[daemon] C7x firmware OK and already running\n"; + return true; + } + + std::cout << "[daemon] Waiting for C7x to reach 'running' state (up to " + << BOOT_TIMEOUT_S << "s)...\n"; + for (int i = 0; i < BOOT_TIMEOUT_S; ++i) { + ::sleep(1); + if (read_sysfs(RPROC_STATE) == "running") { + std::cout << "[daemon] C7x is running\n"; + return true; + } + } + std::cerr << "[daemon] Timeout: C7x did not reach 'running' state\n"; + return false; +} + namespace { volatile sig_atomic_t g_running = 1; @@ -134,6 +229,11 @@ int main(int argc, char* argv[]) { std::cout << "[daemon] Loading TVM artifacts from: " << artifacts << '\n'; + if (!wait_for_c7x()) { + std::cerr << "[daemon] C7x not ready — aborting\n"; + return 1; + } + TvmInferenceClient tvm; tvm.disable_daemon(); /* must not try to connect to itself */ tvm.set_input_shape({1, 2, 401, 161}); @@ -171,9 +271,7 @@ int main(int argc, char* argv[]) { if (g_running) perror("accept"); break; } - std::cout << "[daemon] Client connected\n"; handle_client(cfd, tvm); - std::cout << "[daemon] Client done\n"; } ::unlink(TvmDaemon::SOCKET_PATH);