diff --git a/.gitignore b/.gitignore index 7808e193e..a9ecf2040 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,7 @@ /.vs/ /.vscode/ /nppBackup - +CMakeUserPresets.json # Coverage @@ -140,3 +140,5 @@ poetry.toml /.windsurf/ # emscripten a.out.* + +pkg-adb/ diff --git a/CMakeLists.txt b/CMakeLists.txt index f55128a0a..2cff59da4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE}) option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT}) +option(IGNITE_USE_SYSTEM_DVFS "Use system DVFS library" ON) # 3rd party libs option(LLAMA_HTTPLIB "llama: httplib for downloading functionality" ON) @@ -248,6 +249,9 @@ set_target_properties(llama PUBLIC_HEADER "${LLAMA_PUBLIC_HEADERS}") install(TARGETS llama LIBRARY PUBLIC_HEADER) +# llama-ignite-npu links against dvfs at runtime, so package installs need to +# ship the library alongside the rest of the shared objects. +install(TARGETS dvfs LIBRARY RUNTIME ARCHIVE) configure_package_config_file( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/llama-config.cmake.in diff --git a/common/arg.cpp b/common/arg.cpp index ba27c664d..09adcc66a 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1039,7 +1039,91 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, int value) { params.max_query_number = value; } - ).set_examples({LLAMA_EXAMPLE_SERVER})); + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + + // ---------------------------------------------------------------------------------------- + // 20260406 IGNITE DVFS + // ---------------------------------------------------------------------------------------- + add_opt(common_arg( + {"--device-name", "--dvfs-device"}, "DN", + "DVFS target device name (e.g. S25, S24, Pixel9)", + [](common_params & params, const std::string & value) { + params.device_name = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--cpu-p"}, "IDX", + "prefill CPU DVFS index", + [](common_params & params, int value) { + params.cpu_clk_idx_p = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--ram-p"}, "IDX", + "prefill RAM DVFS index", + [](common_params & params, int value) { + params.ram_clk_idx_p = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--cpu-d"}, "IDX", + "decode CPU DVFS index", + [](common_params & params, int value) { + params.cpu_clk_idx_d = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--ram-d"}, "IDX", + "decode RAM DVFS index", + [](common_params & params, int value) { + params.ram_clk_idx_d = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--phase-pause"}, "MS", + "pause time between prefill and decode phases in milliseconds", + [](common_params & params, int value) { + params.phase_pause = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--token-pause"}, "MS", + "pause time between generated decode tokens in milliseconds", + [](common_params & params, int value) { + params.token_pause = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--layer-pause"}, "MS", + "pause time for layer-wise experiments in milliseconds", + [](common_params & params, int value) { + params.layer_pause = value; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--ignite-verbose"}, + "enable verbose ignite markers for layer pause debugging", + [](common_params & params) { + params.ignite_verbose = true; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--backend-compute-profile"}, + "enable backend scheduler compute profiling in ignite CSV output", + [](common_params & params) { + params.backend_compute_profile = true; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + add_opt(common_arg( + {"--backend-op-breakdown"}, + "append per-op backend scheduler counters to ignite CSV output", + [](common_params & params) { + params.backend_compute_profile = true; + params.backend_op_breakdown = true; + } + ).set_examples({LLAMA_EXAMPLE_COMPLETION})); + + // ---------------------------------------------------------------------------------------- add_opt(common_arg( {"--strict"}, "ST", "enable strict mode", diff --git a/common/common.cpp b/common/common.cpp index 3aa396127..d57df5060 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -374,6 +374,46 @@ void common_init() { LOG_INF("build: %d (%s) with %s for %s%s\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT, LLAMA_COMPILER, LLAMA_BUILD_TARGET, build_type); } +void common_ignite_init(llama_context * ctx, common_params & params) { + if (!ctx) { + return; + } + + llama_igparams ig{}; + + llama_ignite_set_active(ctx, params.is_ignite_active); + llama_ignite_set_layer_pause(ctx, params.layer_pause); + + ig.max_query_number = params.max_query_number; + ig.strict_limit = params.strict_limit; + ig.strict_limit_length = params.strict_limit_length; + ig.enable_thinking = params.enable_thinking; + ig.layer_pause = params.layer_pause; + ig.phase_pause = params.phase_pause; + ig.token_pause = params.token_pause; + ig.query_interval = params.query_interval; + ig.prefill_phase = params.prefill_phase; + ig.prefill_speed = params.prefill_speed; + ig.decode_speed = params.decode_speed; + ig.backend_compute_profile = params.backend_compute_profile; + ig.backend_op_breakdown = params.backend_op_breakdown; + + std::strcpy(ig.input_path, params.input_path.c_str()); + std::strcpy(ig.output_dir, params.output_dir.c_str()); + std::strcpy(ig.output_path_hard, params.output_path_hard.c_str()); + std::strcpy(ig.output_path_infer, params.output_path_infer.c_str()); + + std::strcpy(ig.device_name, params.device_name.c_str()); + ig.is_ignite_active = params.is_ignite_active; + ig.ignite_verbose = params.ignite_verbose; + ig.cpu_clk_idx_p = params.cpu_clk_idx_p; + ig.ram_clk_idx_p = params.ram_clk_idx_p; + ig.cpu_clk_idx_d = params.cpu_clk_idx_d; + ig.ram_clk_idx_d = params.ram_clk_idx_d; + + init_ignite_params(ctx, &ig); +} + std::string common_params_get_system_info(const common_params & params) { std::ostringstream os; @@ -1225,6 +1265,7 @@ common_init_result_ptr common_init_from_params(common_params & params) { LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); return res; } + common_ignite_init(lctx, params); const llama_vocab * vocab = llama_model_get_vocab(model); diff --git a/common/common.h b/common/common.h index c8a67b36f..4581b8f5a 100644 --- a/common/common.h +++ b/common/common.h @@ -621,15 +621,28 @@ struct common_params { bool enable_thinking = false; // llm plane - // int phase_pause = 0; // ms - // int token_pause = 0; // ms - // int layer_pause = 0; // ms - // int query_interval = 0; // ms - // bool prefill_phase = true; // prefill phase or not - // double prefill_speed = 0.0; // tokens/s - // double decode_speed = 0.0; // tokens/s - // bool is_ignite_active = false; - // bool ignite_verbose = false; + std::string device_name = "S25"; + int cpu_clk_idx_p = -1; + int ram_clk_idx_p = -1; + int cpu_clk_idx_d = -1; + int ram_clk_idx_d = -1; + + int phase_pause = 0; // ms + int token_pause = 0; // ms + int layer_pause = 0; // ms + bool backend_compute_profile = false; + bool backend_op_breakdown = false; + int query_interval = 0; // ms + bool prefill_phase = true; // prefill phase or not + double prefill_speed = 0.0; // tokens/s + double decode_speed = 0.0; // tokens/s +#if defined (IGNITE_USE_SYSTEM_DVFS) + bool is_ignite_active = true; + bool ignite_verbose = false; +#else + bool is_ignite_active = false; + bool ignite_verbose = false; +#endif // basic measure configs int max_query_number = -1; // limit of CSV questions (0=no limit) // deprecated in future @@ -643,6 +656,7 @@ struct common_params { // call once at the start of a program if it uses libcommon // initializes the logging system and prints info about the build void common_init(); +void common_ignite_init(llama_context * ctx, common_params & params); std::string common_params_get_system_info(const common_params & params); diff --git a/docs/backend/hexagon/CMakeUserPresets.json b/docs/backend/hexagon/CMakeUserPresets.json new file mode 100644 index 000000000..1f2676c0b --- /dev/null +++ b/docs/backend/hexagon/CMakeUserPresets.json @@ -0,0 +1,51 @@ +{ + "version": 4, + "configurePresets": [ + { + "name": "arm64-android-snapdragon", + "hidden": true, + "architecture": { "value": "arm64", "strategy": "external" }, + "toolset": { "value": "host=x86_64", "strategy": "external" }, + "cacheVariables": { + "ANDROID_ABI": "arm64-v8a", + "ANDROID_PLATFORM": "android-31", + "CMAKE_TOOLCHAIN_FILE": "$env{ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake", + "CMAKE_C_FLAGS": "-march=armv8.7a+fp16 -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE", + "CMAKE_CXX_FLAGS": "-march=armv8.7a+fp16 -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE", + "CMAKE_C_FLAGS_RELEASE": "-O3 -DNDEBUG", + "CMAKE_CXX_FLAGS_RELEASE": "-O3 -DNDEBUG", + "CMAKE_C_FLAGS_RELWITHDEBINFO": "-O3 -DNDEBUG -g", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-O3 -DNDEBUG -g", + "HEXAGON_SDK_ROOT": "$env{HEXAGON_SDK_ROOT}", + "PREBUILT_LIB_DIR": "android_aarch64", + "GGML_OPENMP": "OFF", + "GGML_LLAMAFILE": "OFF", + "GGML_OPENCL": "ON", + "GGML_HEXAGON": "ON", + "GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE": "128", + "LLAMA_OPENSSL": "OFF" + } + }, + + { + "name": "arm64-windows-snapdragon", + "inherits": [ "base", "arm64-windows-llvm" ], + "cacheVariables": { + "HEXAGON_SDK_ROOT": "$env{HEXAGON_SDK_ROOT}", + "PREBUILT_LIB_DIR": "windows_aarch64", + "GGML_OPENMP": "OFF", + "GGML_LLAMAFILE": "OFF", + "GGML_OPENCL": "ON", + "GGML_HEXAGON": "ON", + "GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE": "128", + "LLAMA_OPENSSL": "OFF" + } + }, + + { "name": "arm64-android-snapdragon-debug" , "inherits": [ "base", "arm64-android-snapdragon", "debug" ] }, + { "name": "arm64-android-snapdragon-release", "inherits": [ "base", "arm64-android-snapdragon", "release" ] }, + + { "name": "arm64-windows-snapdragon-debug" , "inherits": [ "base", "arm64-windows-snapdragon", "debug" ] }, + { "name": "arm64-windows-snapdragon-release", "inherits": [ "base", "arm64-windows-snapdragon", "release" ] } + ] +} diff --git a/docs/backend/hexagon/README.md b/docs/backend/hexagon/README.md new file mode 100644 index 000000000..3befdf722 --- /dev/null +++ b/docs/backend/hexagon/README.md @@ -0,0 +1,243 @@ +# Snapdragon-based Android devices + +## How to Build + +The easiest way to build llama.cpp for a Snapdragon-based Android device is using the toolchain Docker image (see github.com/snapdragon-toolchain). +This image includes Android NDK, OpenCL SDK, Hexagon SDK, CMake, etc. + +This method works on Linux, macOS, and Windows. macOS and Windows users should install Docker Desktop. + +``` +~/src/llama.cpp$ docker run -it -u $(id -u):$(id -g) --volume $(pwd):/workspace --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-android:v0.3 +[d]/> cd /workspace +``` + +The rest of the Android build process assumes that you're running inside the toolchain container. +Let's build llama.cpp with CPU, OpenCL, and Hexagon backends via CMake presets: + +``` +[d]/workspace> cp docs/backend/hexagon/CMakeUserPresets.json . + +[d]/workspace> cmake --preset arm64-android-snapdragon-release -B build-snapdragon +Preset CMake variables: + ANDROID_ABI="arm64-v8a" + ... + CMAKE_TOOLCHAIN_FILE="/opt/android-ndk-r28b/build/cmake/android.toolchain.cmake" + GGML_HEXAGON="ON" + GGML_OPENCL="ON" + GGML_OPENMP="OFF" + HEXAGON_SDK_ROOT="/opt/hexagon/6.4.0.2" +... +-- Including OpenCL backend +-- Including Hexagon backend +... +-- Build files have been written to: /workspace/build-snapdragon + +[d]/workspace> cmake --build build-snapdragon +... +[144/356] Performing build step for 'htp-v73' +[1/16] Generating htp_iface_skel.c, htp_iface_stub.c, htp_iface.h +[2/16] Building C object CMakeFiles/ggml-htp-v73.dir/hvx-sigmoid.c.obj +[3/16] Building C object CMakeFiles/ggml-htp-v73.dir/htp-dma.c.obj +[4/16] Building C object CMakeFiles/ggml-htp-v73.dir/worker-pool.c.obj +... +-- Installing: /workspace/build-snapdragon/ggml/src/ggml-hexagon/libggml-htp-v73.so +-- Installing: /workspace/build-snapdragon/ggml/src/ggml-hexagon/libggml-htp-v75.so +... +``` + +To generate an installable "package" simply use cmake --install: + +``` +[d]/workspace> cmake --install build-snapdragon --prefix pkg-adb/llama.cpp +-- Install configuration: "Release" +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-cpu.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-opencl.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-hexagon.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-htp-v73.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-htp-v75.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-htp-v79.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml-htp-v81.so +-- Installing: /workspace/pkg-adb/llama.cpp/lib/libggml.so +... +-- Installing: /workspace/pkg-adb/llama.cpp/bin/llama-bench +-- Installing: /workspace/pkg-adb/llama.cpp/bin/llama-cli +... +``` + +## How to Install + +For this step, your device needs to be configured for on-device development. +Please see https://developer.android.com/studio/debug/dev-options for details. + +Once ADB is enabled, use `adb push` to install `pkg-snapdragon` on the device. +**Note that the toolchain Docker image doesn't have ADB and doesn't set up the ADB bridge. Please use native ADB on the host.** + +``` +~/src/llama.cpp$ adb push pkg-adb/llama.cpp /data/local/tmp/ +pkg-adb/llama.cpp/bin/: 67 files pushed, 0 skipped. 190.2 MB/s (919095042 bytes in 4.607s) +pkg-adb/llama.cpp/include/: 19 files pushed, 0 skipped. 20.5 MB/s (255173 bytes in 0.012s) +pkg-adb/llama.cpp/lib/: 16 files pushed, 0 skipped. 144.4 MB/s (43801382 bytes in 0.289s) +102 files pushed, 0 skipped. 186.9 MB/s (963151597 bytes in 4.914s) +``` + +At this point, you should also install some models: + +``` +~/src/llama.cpp$ wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf +... +2025-10-11 12:04:52 (10.7 MB/s) - ‘Llama-3.2-1B-Instruct-Q4_0.gguf’ saved [773025920/773025920] + +~/src/llama.cpp$ adb push Llama-3.2-1B-Instruct-Q4_0.gguf /data/local/tmp/gguf +Llama-3.2-1B-Instruct-Q4_0.gguf: 1 file pushed, 0 skipped. 38.3 MB/s (773025920 bytes in 19.250s) +``` + +## How to Run + +The easiest way to run llama.cpp cli tools is using provided wrapper scripts that properly set up all required environment variables. + +llama.cpp supports three backends on Snapdragon-based devices: CPU, Adreno GPU (GPUOpenCL), and Hexagon NPU (HTP0-4). +You can select which backend to run the model on using the `D=` variable, which maps to the `--device` option. + +Hexagon NPU behaves as a "GPU" device when it comes to `-ngl` and other offload-related options. + +Here are some examples of running various llama.cpp tools via ADB. + +Simple question for Llama-3.2-1B + +``` +~/src/llama.cpp$ M=Llama-3.2-1B-Instruct-Q4_0.gguf D=HTP0 ./scripts/snapdragon/adb/run-completion.sh -p "what is the most popular cookie in the world?" +... +ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 +ggml-hex: Hexagon Arch version v79 +ggml-hex: allocating new session: HTP0 +ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v79.so?htp_iface_skel_handle_invoke&_modver=1.0&_dom=cdsp&_session=0 handle 0xb4000072c7955e50 +... +load_tensors: offloading output layer to GPU +load_tensors: offloaded 17/17 layers to GPU +load_tensors: CPU model buffer size = 225.49 MiB +load_tensors: HTP0 model buffer size = 0.26 MiB +load_tensors: HTP0-REPACK model buffer size = 504.00 MiB +... +I hope this helps you understand the world's most popular cookies! [end of text] +... +llama_perf_sampler_print: sampling time = 30.08 ms / 487 runs ( 0.06 ms per token, 16191.77 tokens per second) +llama_perf_context_print: load time = 617.94 ms +llama_perf_context_print: prompt eval time = 80.76 ms / 11 tokens ( 7.34 ms per token, 136.21 tokens per second) +llama_perf_context_print: eval time = 9210.59 ms / 475 runs ( 19.39 ms per token, 51.57 tokens per second) +llama_perf_context_print: total time = 9454.92 ms / 486 tokens +llama_perf_context_print: graphs reused = 473 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 439 = 225 + 136 + 77 | +llama_memory_breakdown_print: | - HTP0-REPACK | 504 = 504 + 0 + 0 | +``` + +Summary request for OLMoE-1B-7B. This is a large model that requires two HTP sessions/devices + +``` +~/src/llama.cpp$ M=OLMoE-1B-7B-0125-Instruct-Q4_0.gguf NDEV=2 D=HTP0,HTP1 ./scripts/snapdragon/adb/run-completion.sh -f surfing.txt +... +ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 +ggml-hex: Hexagon Arch version v81 +ggml-hex: allocating new session: HTP0 +ggml-hex: allocating new session: HTP1 +... +load_tensors: offloading output layer to GPU +load_tensors: offloaded 17/17 layers to GPU +load_tensors: CPU model buffer size = 143.86 MiB +load_tensors: HTP1 model buffer size = 0.23 MiB +load_tensors: HTP1-REPACK model buffer size = 1575.00 MiB +load_tensors: HTP0 model buffer size = 0.28 MiB +load_tensors: HTP0-REPACK model buffer size = 2025.00 MiB +... +llama_context: CPU output buffer size = 0.19 MiB +llama_kv_cache: HTP1 KV buffer size = 238.00 MiB +llama_kv_cache: HTP0 KV buffer size = 306.00 MiB +llama_kv_cache: size = 544.00 MiB ( 8192 cells, 16 layers, 1/1 seqs), K (q8_0): 272.00 MiB, V (q8_0): 272.00 MiB +llama_context: HTP0 compute buffer size = 15.00 MiB +llama_context: HTP1 compute buffer size = 15.00 MiB +llama_context: CPU compute buffer size = 24.56 MiB +... +llama_perf_context_print: prompt eval time = 1730.57 ms / 212 tokens ( 8.16 ms per token, 122.50 tokens per second) +llama_perf_context_print: eval time = 5624.75 ms / 257 runs ( 21.89 ms per token, 45.69 tokens per second) +llama_perf_context_print: total time = 7377.33 ms / 469 tokens +llama_perf_context_print: graphs reused = 255 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 742 = 144 + 544 + 54 | +llama_memory_breakdown_print: | - HTP1-REPACK | 1575 = 1575 + 0 + 0 | +llama_memory_breakdown_print: | - HTP0-REPACK | 2025 = 2025 + 0 + 0 | +``` + +Op test for MUL_MAT + +``` +~/src/llama.cpp$ HB=0 ./scripts/snapdragon/adb/run-tool.sh test-backend-ops -b HTP0 -o MUL_MAT +... +Backend 2/3: HTP0 +Device description: Hexagon +Device memory: 2048 MB (2048 MB free) +MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK +MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK +MUL_MAT(type_a=q4_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],v=0,o=1): OK + +~/src/llama.cpp-hexagon$ M=Llama-3.2-1B-Instruct-Q4_0.gguf ./scripts/snapdragon/adb/run-bench.sh -p 128 -n 64 +... +ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev 1 +ggml-hex: Hexagon Arch version v79 +ggml-hex: allocating new session: HTP0 +ggml-hex: new session: HTP0 : session-id 0 domain-id 3 uri file:///libggml-htp-v79.so?htp_iface_skel_handle_invoke&_modver=1.0&_dom=cdsp&_session=0 handle 0xb400007d4b231090 +| model | size | params | backend | ngl | threads | n_batch | mmap | test | t/s | +| ---------------| ---------: | -----: | ---------- | --: | ------: | ------: | ---: | ----: | ------------: | +| llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | pp128 | 169.42 ± 1.75 | +| llama 1B Q4_0 | 729.75 MiB | 1.24 B | HTP | 99 | 4 | 128 | 0 | tg64 | 51.54 ± 1.13 | + +build: 6a8cf8914 (6733) +``` + +## Environment variables + +- `GGML_HEXAGON_NDEV=1` + Controls the number of devices/sessions to allocate. The default is 1. + Most quantized models under 4B fit into a single session; an 8B model needs two, and a 20B model needs four. + +- `GGML_HEXAGON_NHVX=0` + Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version). + +- `GGML_HEXAGON_HOSTBUF=1` + Controls whether the Hexagon backend allocates host buffers. By default, all buffers except for REPACK are host buffers. + This option is required for testing Ops that require REPACK buffers (MUL_MAT and MUL_MAT_ID). + +- `GGML_HEXAGON_EXPERIMENTAL=1` + Controls whether the Hexagon backend enables experimental features. + This option is required for enabling/testing experimental Ops (FLASH_ATTN_EXT). + +- `GGML_HEXAGON_VERBOSE=1` + Enables verbose logging of Ops from the backend. Example output: + + ``` + ggml-hex: HTP0 graph-compute n_nodes 2 + ggml-hex: HTP0 matmul : blk.27.ffn_up.weight x ffn_norm-27 -> ffn_up-27 : 3072:8192 x 3072:1 -> 8192:1 : q4_0 x f32 -> f32 : HTP0 x HTP0 -> HTP0 : flags 0x1 + ggml-hex: HTP0 matmul : blk.27.ffn_gate.weight x ffn_norm-27 -> ffn_gate-27 : 3072:8192 x 3072:1 -> 8192:1 : q4_0 x f32 -> f32 : HTP0 x HTP0 -> HTP0 : flags 0x3 + ggml-hex: HTP0 graph-compute n_nodes 1 + ggml-hex: HTP0 matmul : blk.27.ffn_down.weight x ffn_gate_par-27 -> ffn_out-27 : 8192:3072 x 8192:1 -> 3072:1 : q4_0 x f32 -> f32 : HTP0 x HTP0 -> HTP0 : flags 0x0 + ggml-hex: HTP0 get-tensor result_output : data 0x7592487000 offset 0 size 513024 + ``` + +- `GGML_HEXAGON_PROFILE=1` + Generates a host-side profile for the ggml-hexagon Ops. + +- `GGML_HEXAGON_OPMASK=0x0` + Allows enabling specific stages of the processing pipeline: + + - `0x1` Enable Op Queue (i.e., queuing Ops into NPU) + - `0x2` Enable Dynamic Quantizer (if needed for the Op) + - `0x4` Enable Op Compute (MUL_MAT, etc.) + + Examples: + + `GGML_HEXAGON_OPMASK=0x1 llama-completion ...` - Ops are enqueued but NPU-side processing is stubbed out + `GGML_HEXAGON_OPMASK=0x3 llama-completion ...` - NPU performs dynamic quantization and skips the rest + `GGML_HEXAGON_OPMASK=0x7 llama-completion ...` - Full queuing and processing of Ops (default) diff --git a/docs/backend/hexagon/developer.md b/docs/backend/hexagon/developer.md new file mode 100644 index 000000000..fc4d160e9 --- /dev/null +++ b/docs/backend/hexagon/developer.md @@ -0,0 +1,109 @@ +# Hexagon backend developer details + +## Backend libraries + +The Hexagon backend consist of two parts: + + - `libggml-hexagon` + This is the regular CPU-side GGML backend library, either shared or statically linked + + - `libggml-htp-vNN` + This is the NPU-side (HTP stands for Hexagon Tensor Processor) shared library that contains the Op dispatcher and kernels. + The correct library is selected automatically at runtime based on the HW version. + +Here is an example of the build artifacts + +``` +~/src/llama.cpp$ ls -l pkg-adb/llama.cpp/lib/libggml* +pkg-adb/llama.cpp/lib/libggml-base.so +pkg-adb/llama.cpp/lib/libggml-cpu.so +pkg-adb/llama.cpp/lib/libggml-hexagon.so <<< CPU library +pkg-adb/llama.cpp/lib/libggml-htp-v73.so <<< HTP op/kernels for Hexagon v73 +pkg-adb/llama.cpp/lib/libggml-htp-v75.so +pkg-adb/llama.cpp/lib/libggml-htp-v79.so +pkg-adb/llama.cpp/lib/libggml-htp-v81.so +``` + +## Memory buffers + +Hexagon NPU backend takes advantage of the Snapdragon's unified memory model where all buffers are fully accessible by the CPU and GPU. +The NPU does have a dedicated tightly-coupled memory called VTCM but that memory is used only for intermediate data (e.g. dynamically +quantized tensors) or temporary data (chunks of the weight tensors fetched via DMA). + +Please note that currently the Hexagon backend does not implement SET/GET_ROWS Ops because there is no advantage in offloading those +to the NPU at this point. + +The backend does allocates non-host buffers for the tensors with datatypes that require repacking: Q4_0, Q8_0, MXFP4. +From the MMU perspective these buffers are still regular buffers (normal access by the CPU) they are marked as non-host simply to force +the repacking. + +## Large model handling + +Hexagon NPU session (aka Process Domain (PD) in the Hexagon docs) is limited to a memory mapping of around 3.5GB. +In llama.cpp/GGML the Hexagon session is mapped to a single GGML backend device (HTP0, HTP1, etc). + +In order to map models larger than 3.5GB we need to allocate multiple devices and split the model. +For this we're taking advantage of the llama.cpp/GGML multi-GPU layer-splitting support. +Each Hexagon device behaves like a GPU from the offload and model splitting perspective. + +Here is an example of running GPT-OSS-20B model on a newer Snapdragon device with 16GB of DDR. + +``` +M=gpt-oss-20b-Q4_0.gguf NDEV=4 D=HTP0,HTP1,HTP2,HTP3 P=surfing.txt scripts/snapdragon/adb/run-completion.sh -f surfing.txt -n 32 +... +LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib +ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib +GGML_HEXAGON_NDEV=4 ./bin/llama-cli --no-mmap -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf + -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 --device HTP0,HTP1,HTP2,HTP3 -no-cnv -f surfing.txt +... +llama_model_loader: - type f32: 289 tensors +llama_model_loader: - type q4_0: 96 tensors +llama_model_loader: - type q8_0: 2 tensors +llama_model_loader: - type mxfp4: 72 tensors +... +load_tensors: offloaded 25/25 layers to GPU +load_tensors: CPU model buffer size = 1182.09 MiB +load_tensors: HTP1 model buffer size = 6.64 MiB +load_tensors: HTP1-REPACK model buffer size = 2505.94 MiB +load_tensors: HTP3 model buffer size = 5.55 MiB +load_tensors: HTP3-REPACK model buffer size = 2088.28 MiB +load_tensors: HTP0 model buffer size = 7.75 MiB +load_tensors: HTP0-REPACK model buffer size = 2923.59 MiB +load_tensors: HTP2 model buffer size = 6.64 MiB +load_tensors: HTP2-REPACK model buffer size = 2505.94 MiB +... +llama_context: n_ctx_per_seq (8192) < n_ctx_train (131072) -- the full capacity of the model will not be utilized +llama_context: CPU output buffer size = 0.77 MiB +llama_kv_cache_iswa: creating non-SWA KV cache, size = 8192 cells +llama_kv_cache: HTP1 KV buffer size = 25.50 MiB +llama_kv_cache: HTP3 KV buffer size = 25.50 MiB +llama_kv_cache: HTP0 KV buffer size = 25.50 MiB +llama_kv_cache: HTP2 KV buffer size = 25.50 MiB +llama_kv_cache: size = 102.00 MiB ( 8192 cells, 12 layers, 1/1 seqs), K (q8_0): 51.00 MiB, V (q8_0): 51.00 MiB +llama_kv_cache_iswa: creating SWA KV cache, size = 256 cells +llama_kv_cache: HTP1 KV buffer size = 0.80 MiB +llama_kv_cache: HTP3 KV buffer size = 0.53 MiB +llama_kv_cache: HTP0 KV buffer size = 1.06 MiB +llama_kv_cache: HTP2 KV buffer size = 0.80 MiB +llama_kv_cache: size = 3.19 MiB ( 256 cells, 12 layers, 1/1 seqs), K (q8_0): 1.59 MiB, V (q8_0): 1.59 MiB +llama_context: HTP0 compute buffer size = 16.06 MiB +llama_context: HTP1 compute buffer size = 16.06 MiB +llama_context: HTP2 compute buffer size = 16.06 MiB +llama_context: HTP3 compute buffer size = 16.06 MiB +llama_context: CPU compute buffer size = 98.19 MiB +... +llama_perf_context_print: prompt eval time = 3843.67 ms / 197 tokens ( 19.51 ms per token, 51.25 tokens per second) +llama_perf_context_print: eval time = 1686.13 ms / 31 runs ( 54.39 ms per token, 18.39 tokens per second) +llama_perf_context_print: total time = 6266.30 ms / 228 tokens +llama_perf_context_print: graphs reused = 30 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - HTP0 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP1 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP2 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - HTP3 (Hexagon) | 2048 = 2048 + ( 0 = 0 + 0 + 0) + 0 | +llama_memory_breakdown_print: | - Host | 1476 = 1208 + 105 + 162 | +llama_memory_breakdown_print: | - HTP1-REPACK | 2505 = 2505 + 0 + 0 | +llama_memory_breakdown_print: | - HTP3-REPACK | 2088 = 2088 + 0 + 0 | +llama_memory_breakdown_print: | - HTP0-REPACK | 2923 = 2923 + 0 + 0 | +llama_memory_breakdown_print: | - HTP2-REPACK | 2505 = 2505 + 0 + 0 | +``` diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index a9d177864..60abffabd 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -340,6 +340,74 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // + // Backend scheduler profiling (lightweight) + // + // Notes: + // - This is intended for coarse per-backend and per-phase (prefill/decode) stats. + // - Timing buckets combine scheduler-side wall-time with caller-reported + // build/sampling/process-CPU contributions. + // - The implementation is currently process-global and not thread-safe. + // + + enum ggml_backend_sched_profile_phase { + GGML_BACKEND_SCHED_PROFILE_PREFILL = 0, + GGML_BACKEND_SCHED_PROFILE_DECODE = 1, + }; + + struct ggml_backend_sched_profile_data { + // Unique layer ids observed for ops executed on each backend bucket. + // A layer may be counted in both CPU and HTP if ops for that layer ran on both. + uint32_t prefill_cpu_layers; + uint32_t prefill_htp_layers; + double prefill_cpu_ms; + double prefill_htp_ms; + + uint32_t decode_cpu_layers; + uint32_t decode_htp_layers; + double decode_cpu_ms; + double decode_htp_ms; + + // Operation counts (ggml graph nodes computed). + uint64_t total_ops; + uint64_t prefill_cpu_ops; + uint64_t decode_cpu_ops; + uint64_t prefill_htp_ops; + uint64_t decode_htp_ops; + + // Operation counts by ggml op type. + // Indexed by enum ggml_op, see GGML_OP_* / GGML_OP_COUNT. + uint64_t prefill_cpu_ops_by_type[GGML_OP_COUNT]; + uint64_t decode_cpu_ops_by_type[GGML_OP_COUNT]; + uint64_t prefill_htp_ops_by_type[GGML_OP_COUNT]; + uint64_t decode_htp_ops_by_type[GGML_OP_COUNT]; + + // Overhead breakdown (wall-time, ms) + double prefill_copy_ms; + double prefill_wait_ms; + double prefill_build_ms; + double prefill_sampling_ms; + + double decode_copy_ms; + double decode_wait_ms; + double decode_build_ms; + double decode_sampling_ms; + + }; + + // Enables/disables process-global backend scheduler profiling. + GGML_API void ggml_backend_sched_profile_set_enabled(bool enabled); + // Clears all accumulated profiling state and resets the active phase to prefill. + GGML_API void ggml_backend_sched_profile_reset(void); + // Selects which phase subsequent scheduler/caller-reported metrics are charged to. + GGML_API void ggml_backend_sched_profile_set_phase(enum ggml_backend_sched_profile_phase phase); + // Adds caller-reported graph build time (ms) for the active phase. + GGML_API void ggml_backend_sched_profile_add_build_ms(double build_ms); + // Adds caller-reported sampling time (ms) for the active phase. + GGML_API void ggml_backend_sched_profile_add_sampling_ms(double sampling_ms); + // Returns a snapshot of the accumulated profiling counters. + GGML_API struct ggml_backend_sched_profile_data ggml_backend_sched_profile_get(void); + // // Utils // diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 22c656996..ce87663cb 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #ifdef __APPLE__ @@ -738,6 +739,275 @@ struct ggml_backend_sched { int debug_prev_graph_size; }; +// +// Backend scheduler profiling (process-global, lightweight) +// +// This profiling path is intentionally coarse-grained: +// - phase split: prefill vs decode +// - backend split: CPU vs HTP +// - attribution: per-graph wall-time buckets and per-op counters +// +// It is currently implemented as process-global mutable state, so it is +// intended for single-run instrumentation and is not thread-safe. + +static ggml_backend_sched_profile_phase g_sched_profile_phase = GGML_BACKEND_SCHED_PROFILE_PREFILL; + +struct ggml_backend_sched_profile_state { + bool enabled = false; + ggml_backend_sched_profile_data out = {}; + + // Layer-id presence maps (index = layer id). These are used to count + // unique layers touched in each phase/backend bucket without double + // counting repeated ops from the same layer. + std::vector prefill_cpu_layers; + std::vector prefill_htp_layers; + std::vector decode_cpu_layers; + std::vector decode_htp_layers; +}; + +static ggml_backend_sched_profile_state g_sched_profile; + +static bool ggml_sched_profile_enabled(void) { + return g_sched_profile.enabled; +} + +// Marks a layer id as seen in the target bucket and increments the unique +// layer count only on the first observation. +static void ggml_sched_profile_mark_layer(std::vector & seen, uint32_t & count, int layer_id) { + if (layer_id < 0) { + return; + } + + const size_t idx = (size_t) layer_id; + if (idx >= seen.size()) { + seen.resize(idx + 1, 0); + } + + if (!seen[idx]) { + seen[idx] = 1; + count++; + } +} + +// Best-effort extraction of a layer id from a tensor name. +// Patterns observed in llama graphs: +// - "blk.." (weights) +// - "...-" (intermediates) +// - "..._l" (kv-cache tensors) +// This is approximate by design and is only used for lightweight profiling. +static int ggml_sched_profile_extract_layer_id(const char * name) { + if (!name || !name[0]) { + return -1; + } + + // 1) blk.. + if (const char * p = strstr(name, "blk.")) { + p += 4; + if (!std::isdigit((unsigned char) *p)) { + return -1; + } + + int v = 0; + while (std::isdigit((unsigned char) *p)) { + v = v * 10 + (*p - '0'); + ++p; + } + + if (*p == '.') { + return v; + } + } + + // 2) ...- (must be at end) + if (const char * dash = strrchr(name, '-')) { + const char * p = dash + 1; + if (p[0] && std::isdigit((unsigned char) p[0])) { + int v = 0; + while (std::isdigit((unsigned char) *p)) { + v = v * 10 + (*p - '0'); + ++p; + } + if (*p == '\0') { + return v; + } + } + } + + // 3) ..._l (must be at end) + if (const char * p = strstr(name, "_l")) { + // take the last occurrence to reduce false positives + const char * last = p; + while ((p = strstr(p + 2, "_l"))) { + last = p; + } + + p = last + 2; + if (p[0] && std::isdigit((unsigned char) p[0])) { + int v = 0; + while (std::isdigit((unsigned char) *p)) { + v = v * 10 + (*p - '0'); + ++p; + } + if (*p == '\0') { + return v; + } + } + } + + return -1; +} + +// Scans a scheduled subgraph and records which layer ids were touched by the +// current phase/backend bucket. A layer may appear in both CPU and HTP buckets +// if execution for that layer is split across backends. +static void ggml_sched_profile_note_layers(const ggml_cgraph & graph, bool is_cpu) { + if (!ggml_sched_profile_enabled()) { + return; + } + + for (int i = 0; i < graph.n_nodes; ++i) { + const ggml_tensor * t = graph.nodes[i]; + const int layer_id = ggml_sched_profile_extract_layer_id(t ? t->name : nullptr); + if (layer_id < 0) { + continue; + } + + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + if (is_cpu) { + ggml_sched_profile_mark_layer(g_sched_profile.prefill_cpu_layers, g_sched_profile.out.prefill_cpu_layers, layer_id); + } else { + ggml_sched_profile_mark_layer(g_sched_profile.prefill_htp_layers, g_sched_profile.out.prefill_htp_layers, layer_id); + } + } else { + if (is_cpu) { + ggml_sched_profile_mark_layer(g_sched_profile.decode_cpu_layers, g_sched_profile.out.decode_cpu_layers, layer_id); + } else { + ggml_sched_profile_mark_layer(g_sched_profile.decode_htp_layers, g_sched_profile.out.decode_htp_layers, layer_id); + } + } + } +} + +// Increments per-op counters for the current phase/backend bucket. +static inline void ggml_sched_profile_add_op_type_count(enum ggml_op op, bool is_cpu) { + if (!ggml_sched_profile_enabled()) { + return; + } + + if (op < 0 || op >= GGML_OP_COUNT) { + return; + } + + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + if (is_cpu) { + g_sched_profile.out.prefill_cpu_ops_by_type[op]++; + } else { + g_sched_profile.out.prefill_htp_ops_by_type[op]++; + } + } else { + if (is_cpu) { + g_sched_profile.out.decode_cpu_ops_by_type[op]++; + } else { + g_sched_profile.out.decode_htp_ops_by_type[op]++; + } + } +} + +// Accumulates ggml op counts for all nodes in the scheduled subgraph. +static inline void ggml_sched_profile_add_graph_op_type_counts(const ggml_cgraph & graph, bool is_cpu) { + if (!ggml_sched_profile_enabled()) { + return; + } + + for (int i = 0; i < graph.n_nodes; ++i) { + const ggml_tensor * t = graph.nodes[i]; + if (t == nullptr) { + continue; + } + ggml_sched_profile_add_op_type_count(t->op, is_cpu); + } +} + +void ggml_backend_sched_profile_set_enabled(bool enabled) { + g_sched_profile.enabled = enabled; + if (!enabled) { + ggml_backend_sched_profile_reset(); + } +} + +void ggml_backend_sched_profile_reset(void) { + g_sched_profile_phase = GGML_BACKEND_SCHED_PROFILE_PREFILL; + g_sched_profile.out = {}; + g_sched_profile.prefill_cpu_layers.clear(); + g_sched_profile.prefill_htp_layers.clear(); + g_sched_profile.decode_cpu_layers.clear(); + g_sched_profile.decode_htp_layers.clear(); +} + +void ggml_backend_sched_profile_set_phase(enum ggml_backend_sched_profile_phase phase) { + if (!ggml_sched_profile_enabled()) { + return; + } + + g_sched_profile_phase = phase; +} + +// Records host-side wall time spent copying tensors for the current phase. +static inline void ggml_sched_profile_add_copy_us(const int64_t dt_us) { + if (!ggml_sched_profile_enabled()) { + return; + } + + const double dt_ms = (double) dt_us / 1000.0; + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + g_sched_profile.out.prefill_copy_ms += dt_ms; + } else { + g_sched_profile.out.decode_copy_ms += dt_ms; + } +} + +// Records host-side wall time spent waiting on backend work for the current phase. +static inline void ggml_sched_profile_add_wait_us(const int64_t dt_us) { + if (!ggml_sched_profile_enabled()) { + return; + } + + const double dt_ms = (double) dt_us / 1000.0; + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + g_sched_profile.out.prefill_wait_ms += dt_ms; + } else { + g_sched_profile.out.decode_wait_ms += dt_ms; + } +} + +void ggml_backend_sched_profile_add_build_ms(double build_ms) { + if (!ggml_sched_profile_enabled()) { + return; + } + + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + g_sched_profile.out.prefill_build_ms += build_ms; + } else { + g_sched_profile.out.decode_build_ms += build_ms; + } +} + +void ggml_backend_sched_profile_add_sampling_ms(double sampling_ms) { + if (!ggml_sched_profile_enabled()) { + return; + } + + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + g_sched_profile.out.prefill_sampling_ms += sampling_ms; + } else { + g_sched_profile.out.decode_sampling_ms += sampling_ms; + } +} + +struct ggml_backend_sched_profile_data ggml_backend_sched_profile_get(void) { + return g_sched_profile.out; +} + #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor) #define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)] #define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)] @@ -1455,6 +1725,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + const bool split_is_cpu = ggml_backend_dev_type(ggml_backend_get_device(split_backend)) == GGML_BACKEND_DEVICE_TYPE_CPU; + + // Mark any layer ids we can extract from node names for this split. + // A layer may be marked in both CPU and HTP buckets if different ops in that layer run on both. + ggml_sched_profile_note_layers(split->graph, split_is_cpu); + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); @@ -1464,17 +1740,35 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s if (input->flags & GGML_TENSOR_FLAG_INPUT) { // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + const int64_t t0_us = ggml_time_us(); ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); } else { + const int64_t t0_us = ggml_time_us(); ggml_backend_synchronize(split_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); + } + + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_tensor_copy(input, input_cpy); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_copy_us(t1_us - t0_us); } - ggml_backend_tensor_copy(input, input_cpy); } else { // wait for the split backend to finish using the input before overwriting it if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + const int64_t t0_us = ggml_time_us(); ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); } else { + const int64_t t0_us = ggml_time_us(); ggml_backend_synchronize(split_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); } // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used @@ -1489,7 +1783,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1]; const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1]; - ggml_backend_synchronize(input_backend); + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_synchronize(input_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); + } // get the ids ggml_tensor * ids_tensor = node->src[2]; @@ -1507,8 +1806,18 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s if (ids_tensor != prev_ids_tensor) { ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t)); - ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor)); - ggml_backend_synchronize(ids_backend); + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor)); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_copy_us(t1_us - t0_us); + } + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_synchronize(ids_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); + } // find the used experts used_ids.clear(); @@ -1531,12 +1840,15 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s const size_t padding = std::min(expert_size, 512); const size_t padding_end = last_id < n_expert - 1 ? padding : 0; + const int64_t t0_us = ggml_time_us(); ggml_backend_tensor_set_async(split_backend, input_cpy, (const uint8_t *)input->data + expert_offset, expert_offset, // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert // this is necessary for MMQ in the CUDA backend expert_size_copy + padding_end); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_copy_us(t1_us - t0_us); }; int id = 0; @@ -1563,26 +1875,84 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } copy_experts(first_id, last_id); } else { - // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events - // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface - if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) { - ggml_backend_synchronize(input_backend); + // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, + // since we handle the synchronization here with multiple copies and events. + bool async_ok = false; + if (split_backend->iface.cpy_tensor_async) { + const int64_t t0_us = ggml_time_us(); + async_ok = split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_copy_us(t1_us - t0_us); + } + + if (!async_ok) { + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_synchronize(input_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); + } if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + const int64_t t0_us = ggml_time_us(); ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); } else { + const int64_t t0_us = ggml_time_us(); ggml_backend_synchronize(split_backend); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_wait_us(t1_us - t0_us); + } + { + const int64_t t0_us = ggml_time_us(); + ggml_backend_tensor_copy(input, input_cpy); + const int64_t t1_us = ggml_time_us(); + ggml_sched_profile_add_copy_us(t1_us - t0_us); } - ggml_backend_tensor_copy(input, input_cpy); } } } } + auto prof_add = [&](int n_nodes, int64_t dt_us) { + if (!ggml_sched_profile_enabled()) { + return; + } + + const double dt_ms = (double) dt_us / 1000.0; + g_sched_profile.out.total_ops += (uint64_t) n_nodes; + + ggml_sched_profile_add_graph_op_type_counts(split->graph, split_is_cpu); + + if (g_sched_profile_phase == GGML_BACKEND_SCHED_PROFILE_PREFILL) { + if (split_is_cpu) { + g_sched_profile.out.prefill_cpu_ops += (uint64_t) n_nodes; + g_sched_profile.out.prefill_cpu_ms += dt_ms; + } else { + g_sched_profile.out.prefill_htp_ops += (uint64_t) n_nodes; + g_sched_profile.out.prefill_htp_ms += dt_ms; + } + } else { + if (split_is_cpu) { + g_sched_profile.out.decode_cpu_ops += (uint64_t) n_nodes; + g_sched_profile.out.decode_cpu_ms += dt_ms; + } else { + g_sched_profile.out.decode_htp_ops += (uint64_t) n_nodes; + g_sched_profile.out.decode_htp_ms += dt_ms; + } + } + }; + if (!sched->callback_eval) { + const bool profile_enabled = ggml_sched_profile_enabled(); + const int64_t t0_us = profile_enabled ? ggml_time_us() : 0; enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph); + const int64_t t1_us = profile_enabled ? ggml_time_us() : 0; if (ec != GGML_STATUS_SUCCESS) { return ec; } + + prof_add(split->graph.n_nodes, t1_us - t0_us); } else { // similar to ggml_backend_compare_graph_backend for (int j0 = 0; j0 < split->graph.n_nodes; j0++) { @@ -1601,6 +1971,8 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1); + const bool profile_enabled = ggml_sched_profile_enabled(); + const int64_t t0_us = profile_enabled ? ggml_time_us() : 0; enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv); if (ec != GGML_STATUS_SUCCESS) { return ec; @@ -1608,6 +1980,9 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s // TODO: pass backend to the callback, then the user can decide if they want to synchronize ggml_backend_synchronize(split_backend); + const int64_t t1_us = profile_enabled ? ggml_time_us() : 0; + + prof_add(gv.n_nodes, t1_us - t0_us); if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) { break; diff --git a/include/llama.h b/include/llama.h index bf4e28a8b..e551b07fe 100644 --- a/include/llama.h +++ b/include/llama.h @@ -61,6 +61,7 @@ extern "C" { struct llama_model; struct llama_context; struct llama_sampler; + struct llama_ignite; // for the future typedef struct llama_memory_i * llama_memory_t; @@ -413,6 +414,52 @@ extern "C" { // lora adapter struct llama_adapter_lora; + // used in IGNITE + typedef struct llama_igparams { + // graph internal parameters + bool is_ignite_active; // ignite active status + bool ignite_verbose; // enable verbose logging for ignite + uint16_t layer_pause; // per-layer pause in milliseconds + bool backend_compute_profile; // enable backend scheduler profiling + bool backend_op_breakdown; // append per-op backend scheduler counters + + // graph external parameters + bool strict_limit; + int strict_limit_length; + bool enable_thinking; + + int phase_pause; // ms + int token_pause; // ms + int query_interval; // ms + bool prefill_phase; // prefill phase or not + double prefill_speed; // tokens/s + double decode_speed; // tokens/s + + int max_query_number; // limit of input questions (0=no limit) // deprecated in future + char output_csv_path[128]; // deprecated in future + char input_path[128]; // path = dir/file.ext + char output_dir[128]; + char output_path_hard[128]; + char output_path_infer[128]; + + char device_name[32]; // device name + int cpu_clk_idx_p; // prefill + cpu + int ram_clk_idx_p; // prefill + ram + int cpu_clk_idx_d; // decode + cpu + int ram_clk_idx_d; // decode + ram + bool fixed_config; + + double time_slot; // s + double temp_threshold; // Celsius + double temp_history[64]; // temperature history + int temp_cap; // max length of temperature history + double temp_alpha; // for EMA + int max_cpu_clk_idx; // fixed by device + int cur_cpu_clk_idx; // dynamic + int max_ram_clk_idx; // fixed by device + int cur_ram_clk_idx; // dynamic + } llama_igparams; + // Helpers for getting default parameters // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) LLAMA_API struct llama_model_params llama_model_default_params(void); @@ -1007,6 +1054,14 @@ extern "C" { // otherwise: float[n_embd] (1-dimensional) LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + // ignite + LLAMA_API void llama_ignite_set_active(struct llama_context * ctx, bool active); + LLAMA_API bool llama_ignite_get_active(struct llama_context * ctx); + LLAMA_API void llama_ignite_set_layer_pause(struct llama_context * ctx, uint16_t ms); + LLAMA_API bool init_ignite_params(struct llama_context * ctx, llama_igparams * igparams); + LLAMA_API bool init_ignite_filename(struct llama_context * ctx); + LLAMA_API llama_igparams * get_ignite_params(struct llama_context * ctx); + // // backend sampling API [EXPERIMENTAL] // note: use only if the llama_context was created with at least one llama_sampler_seq_config diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 9373fab14..0165c5647 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,8 +1,8 @@ set(TARGET llama-ignite-npu) add_executable(${TARGET} ignite-npu.cpp) -target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) -target_compile_features(${TARGET} PRIVATE cxx_std_17) +target_link_libraries(${TARGET} PRIVATE common llama dvfs ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) if(LLAMA_TOOLS_INSTALL) install(TARGETS ${TARGET} RUNTIME) -endif() +endif() \ No newline at end of file diff --git a/main/ignite-npu.cpp b/main/ignite-npu.cpp index 6159f1357..b07ff7493 100644 --- a/main/ignite-npu.cpp +++ b/main/ignite-npu.cpp @@ -6,14 +6,19 @@ #include "llama.h" #include "chat.h" +#include "ggml-backend.h" + #include #include #include +#include +#include #include #include #include #include #include +#include #include // to accumulate json #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) @@ -34,6 +39,13 @@ #include "nlohmann/json.hpp" + +// dvfs library +#include "hard/record.h" +#include "hard/dvfs.h" +#include "hard/utils.h" +#include "hard/affinity.h" + using json = nlohmann::json; static llama_context ** g_ctx; @@ -45,6 +57,53 @@ static std::ostringstream * g_output_ss; static std::vector * g_output_tokens; static bool is_interacting = false; static bool need_insert_eot = false; +std::atomic_bool sigterm(false); + +static bool env_flag_enabled(const char * name) { + const char * env = std::getenv(name); + if (env == nullptr) { + return false; + } + + return std::strcmp(env, "1") == 0 || std::strcmp(env, "true") == 0 || std::strcmp(env, "TRUE") == 0; +} + +static bool should_write_backend_profile_csv(const llama_igparams * ig) { + return ig != nullptr && ig->backend_compute_profile; +} + +static bool should_write_op_breakdown_csv(const llama_igparams * ig) { + return should_write_backend_profile_csv(ig) && + (ig->backend_op_breakdown || env_flag_enabled("IGNITE_CSV_OP_BREAKDOWN")); +} + +// Appends per-op CSV headers for the optional op breakdown section. +// Each ggml op contributes four columns: +// prefill_cpu, decode_cpu, prefill_htp, decode_htp. +// TODO: move to utils if this CSV formatting is reused outside ignite-npu. +static void append_profile_csv_op_headers(std::ostream & os) { + for (int op = 0; op < GGML_OP_COUNT; ++op) { + const char * op_name = ggml_op_name((ggml_op) op); + if (op_name == nullptr || op_name[0] == '\0') { + op_name = "unknown"; + } + os << ",prefill_cpu_op_" << op_name + << ",decode_cpu_op_" << op_name + << ",prefill_htp_op_" << op_name + << ",decode_htp_op_" << op_name; + } +} + +// Appends per-op CSV values matching append_profile_csv_op_headers(). +// TODO: move to utils if this CSV formatting is reused outside ignite-npu. +static void append_profile_csv_op_values(std::ostream & os, const ggml_backend_sched_profile_data & prof) { + for (int op = 0; op < GGML_OP_COUNT; ++op) { + os << "," << prof.prefill_cpu_ops_by_type[op] + << "," << prof.decode_cpu_ops_by_type[op] + << "," << prof.prefill_htp_ops_by_type[op] + << "," << prof.decode_htp_ops_by_type[op]; + } +} void ctx_kv_cache_clear(struct llama_context * ctx) { //llama_kv_cache_clear(ctx); //deprecated @@ -52,7 +111,7 @@ void ctx_kv_cache_clear(struct llama_context * ctx) { llama_memory_clear(mem, true); } -std::tuple llama_perf_context_print_custom(const struct llama_context * ctx, const std::string & output_filename, std::chrono::time_point start_sys_time) { +std::tuple llama_perf_context_print_custom(const struct llama_context * ctx, const std::string & output_filename, std::chrono::time_point start_sys_time, const llama_igparams * ig) { const auto data = llama_perf_context(ctx); const double t_end_ms = 1e-3 * ggml_time_us(); @@ -63,17 +122,47 @@ std::tuple llama_perf_context_print_custom(const struc // __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval); // LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval)); - // Open the CSV file in append mode - - + // Open the CSV file in append mode. + // The fixed columns store aggregate throughput/timing counters. Backend + // profiling columns are appended only when requested by the ignite options. + // Convert time_point to time_t (seconds since epoch) auto now_sys_time = std::chrono::system_clock::now(); auto sys_time = std::chrono::duration_cast(now_sys_time-start_sys_time).count(); + // system time, prefill speed, decode speed, prefill tokens, decode tokens, ttft std::ofstream file(output_filename, std::ios::app); if (file.is_open()) { file << std::to_string(sys_time) << "," << ( 1e3 / data.t_p_eval_ms *data.n_p_eval ) << "," << (1e3 / data.t_eval_ms * data.n_eval ) << "," - << data.n_p_eval << ","<< data.n_eval << "," << (data.t_p_eval_ms)<<"\n"; + << data.n_p_eval << ","<< data.n_eval << "," << (data.t_p_eval_ms); + if (should_write_backend_profile_csv(ig)) { + const auto prof = ggml_backend_sched_profile_get(); + file << "," << prof.prefill_cpu_layers + << "," << prof.prefill_htp_layers + << "," << prof.prefill_cpu_ms + << "," << prof.prefill_htp_ms + << "," << prof.decode_cpu_layers + << "," << prof.decode_htp_layers + << "," << prof.decode_cpu_ms + << "," << prof.decode_htp_ms + << "," << prof.total_ops + << "," << prof.prefill_cpu_ops + << "," << prof.decode_cpu_ops + << "," << prof.prefill_htp_ops + << "," << prof.decode_htp_ops + << "," << prof.prefill_copy_ms + << "," << prof.prefill_wait_ms + << "," << prof.prefill_build_ms + << "," << prof.prefill_sampling_ms + << "," << prof.decode_copy_ms + << "," << prof.decode_wait_ms + << "," << prof.decode_build_ms + << "," << prof.decode_sampling_ms; + if (should_write_op_breakdown_csv(ig)) { + append_profile_csv_op_values(file, prof); + } + } + file << "\n"; file.close(); } else { // LLAMA_LOG_INFO("Failed to open file: %s\n", output_filename.c_str()); @@ -229,6 +318,17 @@ int main(int argc, char ** argv) { return 1; } + auto * ig = get_ignite_params(ctx); + if (ig == nullptr) { + LOG_ERR("%s: failed to get ignite params\n", __func__); + return 1; + } + if (env_flag_enabled("IGNITE_CSV_OP_BREAKDOWN")) { + ig->backend_compute_profile = true; + ig->backend_op_breakdown = true; + ggml_backend_sched_profile_set_enabled(true); + } + llama_memory_t mem = llama_get_memory(ctx); const llama_vocab * vocab = llama_model_get_vocab(model); @@ -515,10 +615,72 @@ int main(int argc, char ** argv) { auto start_sys_time = std::chrono::system_clock::now(); std::ofstream file(output_path_infer, std::ios::app); if (file.is_open() && output_path_infer!="/inference_stats.csv") { - file << "sys_time, prefill_speed, decode_speed, prefill_token, decode_token, ttft\n"; + file << "sys_time,prefill_speed,decode_speed,prefill_token,decode_token,ttft"; + if (should_write_backend_profile_csv(ig)) { + file << ",prefill_cpu_layers,prefill_htp_layers,prefill_cpu_ms,prefill_htp_ms"; + file << ",decode_cpu_layers,decode_htp_layers,decode_cpu_ms,decode_htp_ms"; + file << ",total_ops,prefill_cpu_ops,decode_cpu_ops,prefill_htp_ops,decode_htp_ops"; + file << ",prefill_copy_ms,prefill_wait_ms,prefill_build_ms,prefill_sampling_ms"; + file << ",decode_copy_ms,decode_wait_ms,decode_build_ms,decode_sampling_ms"; + if (should_write_op_breakdown_csv(ig)) { + append_profile_csv_op_headers(file); + } + } + file << "\n"; file.close(); } + // dummy dvfs object + const std::string device_name = + std::strlen(ig->device_name) > 0 ? ig->device_name : "S25"; + + DVFS dvfs(device_name); + dvfs.control_start_point = start_sys_time; + dvfs.output_filename = params.output_dir + "/hardware_stats.csv"; + + const bool want_prefill_dvfs = ig->cpu_clk_idx_p >= 0 || ig->ram_clk_idx_p >= 0; + const bool want_decode_dvfs = ig->cpu_clk_idx_d >= 0 || ig->ram_clk_idx_d >= 0; + bool runtime_dvfs_ready = false; + + if (want_prefill_dvfs || want_decode_dvfs) { + runtime_dvfs_ready = (dvfs.init_fd_cache() == 0); + if (!runtime_dvfs_ready) { + LOG_WRN("%s: failed to init DVFS for %s, continuing without runtime DVFS\n", + __func__, device_name.c_str()); + } + } + + auto apply_dvfs = [&](int cpu_idx, int ram_idx) { + if (!runtime_dvfs_ready) { + return; + } + + if (cpu_idx >= 0) { + auto conf = dvfs.get_cpu_freqs_conf(cpu_idx); + if (dvfs.set_cpu_freq(conf) != 0) { + LOG_WRN("%s: failed to set CPU DVFS index %d\n", __func__, cpu_idx); + } + } + + if (ram_idx >= 0) { + if (dvfs.set_ram_freq(ram_idx) != 0) { + LOG_WRN("%s: failed to set RAM DVFS index %d\n", __func__, ram_idx); + } + } + }; + + auto reset_dvfs = [&]() { + if (!runtime_dvfs_ready) { + return; + } + dvfs.unset_cpu_freq(); + dvfs.unset_ram_freq(); + }; + + #if IGNITE_USE_SYSTEM_DVFS + std::thread record_thread = std::thread(record_hard, std::ref(sigterm), std::ref(dvfs)); + #endif + // Input json file instead of cli input std::vector json_questions; size_t current_question_index = 0; @@ -530,7 +692,7 @@ int main(int argc, char ** argv) { // } } bool custom_max_query = params.max_query_number == -1 ? false : true; - unsigned int max_query_num = custom_max_query ? params.max_query_number : json_questions.size(); + size_t max_query_num = custom_max_query ? (size_t) params.max_query_number : json_questions.size(); // JSON questions load done //------------------------------------------------ @@ -792,10 +954,32 @@ int main(int argc, char ** argv) { } if (!embd.empty()) { + // prefill/decode detector + if (!generation_started) { + // prefill phase + if (!prefill_active && ig->is_ignite_active) { + apply_dvfs(ig->cpu_clk_idx_p, ig->ram_clk_idx_p); + } + if (!prefill_active) { + prefill_active = true; + decode_active = false; + } + } else { + // decode phase + if (!decode_active && ig->is_ignite_active) { + apply_dvfs(ig->cpu_clk_idx_d, ig->ram_clk_idx_d); + } + if (!decode_active) { + prefill_active = false; + decode_active = true; + } + } int n_eval = (int) embd.size(); + LOG_DBG("eval: %s\n", string_from(ctx, embd).c_str()); GGML_ASSERT(n_eval <= params.n_batch); + if (llama_decode(ctx, llama_batch_get_one(embd.data(), n_eval))) { LOG_ERR("%s : failed to eval\n", __func__); return 1; @@ -819,6 +1003,13 @@ int main(int argc, char ** argv) { embd.clear(); if ((int) embd_inp.size() <= n_consumed && !is_interacting) { + if (!generation_started) { + if (ig->phase_pause > 0) { + std::this_thread::sleep_for( + std::chrono::milliseconds(ig->phase_pause)); + } + } + // ------------------------------------------------ // now, generation starts generation_started = true; @@ -832,9 +1023,13 @@ int main(int argc, char ** argv) { LOG_DBG("saved session to %s\n", path_session.c_str()); } + const int64_t t_sample_us = ig->backend_compute_profile ? ggml_time_us() : 0; const llama_token id = common_sampler_sample(smpl, ctx, -1); common_sampler_accept(smpl, id, /* accept_grammar= */ true); + if (ig->backend_compute_profile) { + ggml_backend_sched_profile_add_sampling_ms((ggml_time_us() - t_sample_us) / 1000.0); + } // LOG_DBG("last: %s\n", string_from(ctx, smpl->prev.to_vector()).c_str()); @@ -983,7 +1178,7 @@ int main(int argc, char ** argv) { // LOG_INF("Inference time for previous question: %lld ms\n", inference_duration); common_perf_print(ctx, smpl); if(output_path_infer!="/inference_stats.csv"){ // deprecated in future - llama_perf_context_print_custom(ctx, output_path_infer, start_sys_time); + llama_perf_context_print_custom(ctx, output_path_infer, start_sys_time, ig); } //check_hardware(device_name); // common_sampler_free(smpl); @@ -1030,24 +1225,31 @@ int main(int argc, char ** argv) { buffer = "/no_think "; // see `general.architecture` auto tmp = json_questions[current_question_index-1]; // only json requires -1 buffer += tmp; - + // context reset for new question ctx_kv_cache_clear(ctx); embd_inp.clear(); llama_perf_context_reset(ctx); + if (ig->backend_compute_profile) { + ggml_backend_sched_profile_reset(); + } n_past = 0; n_consumed = 0; waiting_for_first_input = true; common_sampler_reset(smpl); + // reset dvfs will be not called after query finished + prefill_active = false; + decode_active = false; generation_started = false; + n_remain = params.n_predict; ga_i = 0; is_antiprompt = false; - + // logger info LOG_INF("[%zu/%zu] ", current_question_index, max_query_num); // LOG_INF("Using question from file: %s\n", buffer.c_str()); LOG("%s\n", tmp.c_str()); - + // Record the begining time of inference for a new question inference_start_time = std::chrono::steady_clock::now(); inference_started = true; @@ -1168,6 +1370,13 @@ int main(int argc, char ** argv) { is_interacting = true; } } + + #if IGNITE_USE_SYSTEM_DVFS + sigterm = true; + record_thread.join(); + #endif + + reset_dvfs(); if (!path_session.empty() && params.prompt_cache_all && !params.prompt_cache_ro) { LOG("\n%s: saving final output to session file '%s'\n", __func__, path_session.c_str()); diff --git a/scripts-termux/ignite-qwen.sh b/scripts-termux/ignite-qwen.sh new file mode 100755 index 000000000..317b7c9c7 --- /dev/null +++ b/scripts-termux/ignite-qwen.sh @@ -0,0 +1,107 @@ +# product name +# DEV="$(getprop ro.product.product.model)" +# DEV="$(printf '%s' "$DEV" | tr -d '[:space:]')" +DEV="S25" +echo "Device: $DEV" + +# turn-off screen +if [ "$DEV" = "Pixel9" ]; then + # Pixel9 + su -c "echo 0 > /sys/class/backlight/panel0-backlight/brightness" +elif [ "$DEV" = "S24" ] || [ "$DEV" = "S25" ]; then + # S24, S25 + su -c "echo 0 > /sys/class/backlight/panel0-backlight/brightness" +else + # Default + su -c "echo 0 > /sys/class/backlight/panel/brightness" + DEV="S25" +fi + + +# CPU Governor: performance +su -c "echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor" +su -c "echo performance > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor" +echo "CPU Governor (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor)" +echo "CPU Governor (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor)" +sleep 3 + +# silver core control (Except S25) +if [ "$DEV" != "S25" ]; then + su -c "echo 0 > /sys/devices/system/cpu/cpu1/online" + su -c "echo 0 > /sys/devices/system/cpu/cpu2/online" + su -c "echo 0 > /sys/devices/system/cpu/cpu3/online" +fi + + # -m: model path + # -v: vocabulary path + # -e: merges path + # -f: model family + # -b: model size + # -t: num of threads + # -l: max KV cache size + # -i: print inference interface + # -s: starting num of queries + # -L: num of queries + # -I: input dataset path of csv + # -O: output directory path + # -S: save query-answer pairs with json + # -D: device name + # --strict: apply tokwn limits to only output tokens + # --cpu-p: specify CPU frequency for CPU DVFS + # --ram-p: specify RAM frequency for RAM DVFS + # --cpu-d: specify CPU frequency for CPU DVFS + # --ram-d: specify RAM frequency for RAM DVFS + # --phase-pause: specify a pause time between phases (ms) + # --token-pause: specify a pause time between generation tokens (ms) + # --layer-pause: specify a pause time between self-attention layers during prefill (ms) + # --query-interval: specify an interval time between queries (s) + +./bin-arm/stream_qwen \ + -m models/qwen-1-5-0.5b-q4_k.mllm \ + -v vocab/qwen_vocab.mllm \ + -e vocab/qwen_merges.txt \ + -f Qwen1.5 \ + -b 0.5B \ + -t 4 \ + -l 1024 \ + -i 1 \ + -s 1 \ + -L 30 \ + -I dataset/hotpot_qa.csv \ + -O output/ \ + -S 0 \ + -D "$DEV" \ + --strict 0 \ + --cpu-p $1 \ + --ram-p $2 \ + --cpu-d $3 \ + --ram-d $4 \ + --phase-pause 0 \ + --token-pause 0 \ + --layer-pause 0 \ + --query-interval 0 + +# [pause-unit] = ms +# [interval-unit] = s + +# silver core reset (except S25) +if [ "$DEV" != "S25" ]; then + su -c "echo 1 > /sys/devices/system/cpu/cpu1/online" + su -c "echo 1 > /sys/devices/system/cpu/cpu2/online" + su -c "echo 1 > /sys/devices/system/cpu/cpu3/online" +fi + +# CPU Governor reset: walt +su -c "echo walt > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor" +su -c "echo walt > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor" +echo "CPU Governor reset (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor)" +echo "CPU Governor reset (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor)" + +# turn-on screen +if [ "$DEV" = "S25" ]; then + # S25 + su -c "echo 1023 > /sys/class/backlight/panel0-backlight/brightness" +else + # S24 + su -c "echo 1023 > /sys/class/backlight/panel/brightness" +fi diff --git a/scripts-termux/qwen3_run.sh b/scripts-termux/qwen3_run.sh new file mode 100755 index 000000000..0a8822126 --- /dev/null +++ b/scripts-termux/qwen3_run.sh @@ -0,0 +1,50 @@ +# this script should be run on llama.cpp/ dir. + +# screen brightness control +echo 0 > /sys/class/backlight/panel0-backlight/brightness + +# silver core control +# su -c "echo 1 > /sys/devices/system/cpu/cpu1/online" +# su -c "echo 1 > /sys/devices/system/cpu/cpu2/online" +# su -c "echo 1 > /sys/devices/system/cpu/cpu3/online" + +# CPU Governor: performance +echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +echo performance > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor +echo "CPU Governor (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor)" +echo "CPU Governor (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor)" +sleep 3 + +./build/bin/ignite \ + -m models/qwen-3-0.6b-q4_k_m.gguf \ + -i -cnv -tb 5 -t 5 -ub 512 -b 512 \ + -c 1024 \ + --temp 0 \ + --top-k 1 \ + --device-name S25 \ + --output-dir output/ \ + --backend-compute-profile \ + --input-path data/qwen3_prefill_64.json \ + -fa off \ + --strict on \ + --strict-limit 128 \ + --max-query-number 30 \ + --cpu-p 15 \ + --ram-p 9 \ + --cpu-d 15 \ + --ram-d 9 + +# --layer-pause LP[ms] + +# su -c "echo 1 > /sys/devices/system/cpu/cpu1/online" +# su -c "echo 1 > /sys/devices/system/cpu/cpu2/online" +# su -c "echo 1 > /sys/devices/system/cpu/cpu3/online" + +# CPU Governor reset: walt +echo walt > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +echo walt > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor +echo "CPU Governor reset (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor)" +echo "CPU Governor reset (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor)" + +# experiment done -> let screen brightness bright again +echo 1023 > /sys/class/backlight/panel0-backlight/brightness diff --git a/scripts-termux/run-setup.sh b/scripts-termux/run-setup.sh deleted file mode 100644 index f0c92615e..000000000 --- a/scripts-termux/run-setup.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!bin/bash - -# directory check -if [ -d "~/.cache/llama.cpp" ] -then - echo "model directory is ready" -else - echo "model directory does not exist" - mkdir -p ~/.cache/llama.cpp/ - echo "model directory is ready now" -fi - - -# model check -if [ -e "~/.cache/llama.cpp/tensorblock_Qwen1.5-0.5B-GGUF_Qwen1.5-0.5B-Q4_K.gguf" ] -then - curl -L https://huggingface.co/tensorblock/Qwen1.5-0.5B-GGUF/resolve/main/Qwen1.5-0.5B-Q5_K_M.gguf --output ~/.cache/llama.cpp/tensorblock_Qwen1.5-0.5B-GGUF_Qwen1.5-0.5B-Q4_K.gguf -fi - - -# directory check -if [ -d "outputs" ] -then - echo "ouputs directory is ready" -else - echo "outputs directory does not exist" - mkdir outputs - echo "outputs directory is ready now" -fi \ No newline at end of file diff --git a/scripts-termux/run.sh b/scripts-termux/run.sh index 254b15ecf..f94c9d443 100644 --- a/scripts-termux/run.sh +++ b/scripts-termux/run.sh @@ -1,40 +1,100 @@ -#! bin/bash -# this script should be run on llama.cpp/ dir. +#!/bin/sh +# run.sh - NPU inference with runtime DVFS controlled by llama-ignite-npu +# Run from: /data/local/tmp/llama.cpp +# +# Optional positional args: +# $1: prefill CPU DVFS index +# $2: prefill RAM DVFS index +# $3: decode CPU DVFS index +# $4: decode RAM DVFS index +# $5: phase pause in ms +# $6: token pause in ms +# $7: layer pause in ms +# $8: ignite verbose [on|off] -# screen brightness control -su -c "echo 0 > /sys/class/backlight/panel0-backlight/brightness" +DEV="${DEV:-S25}" +CPU_P="${1:-15}" +RAM_P="${2:-9}" +CPU_D="${3:-15}" +RAM_D="${4:-9}" +PHASE_PAUSE_MS="${5:-0}" +TOKEN_PAUSE_MS="${6:-0}" +LAYER_PAUSE_MS="${7:-0}" +IGNITE_VERBOSE="${8:-off}" -# silver core control -su -c "echo 1 > /sys/devices/system/cpu/cpu1/online" -su -c "echo 1 > /sys/devices/system/cpu/cpu2/online" -su -c "echo 1 > /sys/devices/system/cpu/cpu3/online" +case "$IGNITE_VERBOSE" in + 1|on|ON|true|TRUE|yes|YES) + IGNITE_VERBOSE_ARG="--ignite-verbose" + ;; + *) + IGNITE_VERBOSE_ARG="" + ;; +esac +restore_system_state() { + status=$? -./build/bin/ignite \ - -m ~/.cache/llama.cpp/tensorblock_Qwen1.5-0.5B-GGUF_Qwen1.5-0.5B-Q4_K.gguf \ - -i \ - -cnv \ - -c 1024 \ - --temp 0 \ - --top-k 5 \ - --threads 1 \ - --device-name Pixel9 \ - --ignite-verbose off \ - --output-dir outputs/ \ - --json-path dataset/hotpot_qa_30.json \ - --strict on \ - --strict-length 64 \ - --max-query-number 20 \ - --cpu-p 12 \ - --ram-d 11 \ - --cpu-p 12 \ - --ram-d 11 + echo 1023 > /sys/class/backlight/panel0-backlight/brightness 2>/dev/null || true + echo "[restore] screen on" + echo walt > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor 2>/dev/null || true + echo walt > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor 2>/dev/null || true + echo "CPU Governor reset (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor 2>/dev/null)" + echo "CPU Governor reset (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor 2>/dev/null)" -su -c "echo 1 > /sys/devices/system/cpu/cpu1/online" -su -c "echo 1 > /sys/devices/system/cpu/cpu2/online" -su -c "echo 1 > /sys/devices/system/cpu/cpu3/online" + echo "[inference] done." + + trap - EXIT INT TERM + exit "$status" +} + +trap restore_system_state EXIT INT TERM + +# screen brightness control +echo 0 > /sys/class/backlight/panel0-backlight/brightness -# experiment done -> let screen brightness bright again -su -c "echo 1023 > /sys/class/backlight/panel0-backlight/brightness" +# Keep governor setup in the script, but let ignite-npu control per-phase DVFS. +echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor +echo performance > /sys/devices/system/cpu/cpufreq/policy6/scaling_governor +echo "CPU Governor (policy0): $(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor)" +echo "CPU Governor (policy6): $(cat /sys/devices/system/cpu/cpufreq/policy6/scaling_governor)" +sleep 2 +echo "[setup] DVFS device: $DEV" +echo "[setup] DVFS indices: prefill(cpu=$CPU_P, ram=$RAM_P), decode(cpu=$CPU_D, ram=$RAM_D)" +echo "[setup] Phase pause: ${PHASE_PAUSE_MS}ms" +echo "[setup] Token pause: ${TOKEN_PAUSE_MS}ms" +echo "[setup] Layer pause: ${LAYER_PAUSE_MS}ms" +echo "[setup] Ignite verbose: ${IGNITE_VERBOSE}" + +setenforce 0 || true + +export LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib +export ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib +export GGML_HEXAGON_HOSTBUF=1 + +cd /data/local/tmp/llama.cpp || exit 1 + +taskset fe ./bin/llama-ignite-npu \ + -m /data/local/tmp/gguf/qwen-3-1.7b-q4_k_m.gguf \ + -t 6 -tb 6 -i -cnv -ub 512 -b 512 -fa off \ + --json-path data/qwen3_prefill_64_20.json \ + --max-query-number 20 \ + --strict on \ + --strict-limit 128 \ + --output-dir output \ + --backend-compute-profile \ + --backend-op-breakdown \ + --temp 0 \ + --top-k 1 \ + -c 1024 \ + --device HTP0 \ + --dvfs-device "$DEV" \ + --cpu-p "$CPU_P" \ + --ram-p "$RAM_P" \ + --cpu-d "$CPU_D" \ + --ram-d "$RAM_D" \ + --phase-pause "$PHASE_PAUSE_MS" \ + --token-pause "$TOKEN_PAUSE_MS" \ + --layer-pause "$LAYER_PAUSE_MS" \ + ${IGNITE_VERBOSE_ARG} diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 000000000..e627a2544 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +REPO_ROOT=$(dirname "$SCRIPT_DIR") + +BUILD_DIR=${BUILD_DIR:-"$REPO_ROOT/build"} +BUILD_TYPE=${BUILD_TYPE:-Release} +BUILD_TARGET=${BUILD_TARGET:-llama-ignite-npu} +JOBS=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 4)} + +if [ ! -f "$REPO_ROOT/CMakeLists.txt" ]; then + echo "error: could not find CMakeLists.txt in $REPO_ROOT" >&2 + exit 1 +fi + +cmake -S "$REPO_ROOT" -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE="$BUILD_TYPE" "$@" +cmake --build "$BUILD_DIR" --target "$BUILD_TARGET" --config "$BUILD_TYPE" -j"$JOBS" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f337afd6b..75c92d798 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(llama llama-graph.cpp llama-hparams.cpp llama-impl.cpp + llama-ignite.cpp llama-io.cpp llama-kv-cache.cpp llama-kv-cache-iswa.cpp @@ -36,6 +37,8 @@ add_library(llama unicode-data.cpp unicode.cpp unicode.h + hard/utils.cpp + hard/utils.h models/afmoe.cpp models/apertus.cpp models/arcee.cpp @@ -142,18 +145,44 @@ add_library(llama models/graph-context-mamba.cpp ) +add_library(dvfs + hard/device.cpp + hard/device.h + hard/dvfs.cpp + hard/dvfs.h + hard/record.cpp + hard/record.h + hard/utils.cpp + hard/utils.h + hard/affinity.cpp + hard/affinity.h + ) + set_target_properties(llama PROPERTIES VERSION ${LLAMA_INSTALL_VERSION} SOVERSION 0 MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number ) +# llama library target_include_directories(llama PRIVATE .) target_include_directories(llama PUBLIC ../include) +target_include_directories(llama PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../vendor") target_compile_features (llama PRIVATE cxx_std_17) # don't bump - target_link_libraries(llama PUBLIC ggml) +# dvfs library +target_include_directories(dvfs PRIVATE .) +target_include_directories(dvfs PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(dvfs PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../vendor") +target_compile_features (dvfs PRIVATE cxx_std_17) # don't bump +target_link_libraries(dvfs PUBLIC ggml) +if (IGNITE_USE_SYSTEM_DVFS) + target_compile_definitions(dvfs PUBLIC IGNITE_USE_SYSTEM_DVFS=1) +else() + target_compile_definitions(dvfs PUBLIC IGNITE_USE_SYSTEM_DVFS=0) +endif() + if (BUILD_SHARED_LIBS) set_target_properties(llama PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_definitions(llama PRIVATE LLAMA_BUILD) diff --git a/src/hard/affinity.cpp b/src/hard/affinity.cpp new file mode 100644 index 000000000..bfb4597ab --- /dev/null +++ b/src/hard/affinity.cpp @@ -0,0 +1,16 @@ +#include "affinity.h" + + +static pid_t gettid_() { + return (pid_t)syscall(SYS_gettid); +} + +static void pin_tid(pid_t tid, std::initializer_list cpus) { + cpu_set_t cs; CPU_ZERO(&cs); + for (int c : cpus) CPU_SET(c, &cs); + sched_setaffinity(tid, sizeof(cs), &cs); +} + +void pin_current(std::initializer_list cpus) { + pin_tid(gettid_(), cpus); +} \ No newline at end of file diff --git a/src/hard/affinity.h b/src/hard/affinity.h new file mode 100644 index 000000000..2b3cd0608 --- /dev/null +++ b/src/hard/affinity.h @@ -0,0 +1,15 @@ +#ifndef __AFFINITY_H +#define __AFFINITY_H + +#include // for sched_setaffinity +#include // for syscall(SYS_gettid) +#include // for syscall + +#include // for std::initializer_list + +static pid_t gettid_(); +static void pin_tid(pid_t tid, std::initializer_list cpus); +void pin_current(std::initializer_list cpus); + + +#endif // __AFFINITY_H \ No newline at end of file diff --git a/src/hard/device.cpp b/src/hard/device.cpp new file mode 100644 index 000000000..8e6cef6fb --- /dev/null +++ b/src/hard/device.cpp @@ -0,0 +1,19 @@ +#include "device.h" + +Device::Device(const std::string& device_name) : device(device_name){ + if (device_name == "S22_Ultra" || device_name == "Fold4" || device_name == "Pixel9"){ + cluster_indices = {0, 4, 7}; + } else if (device_name == "S24"){ + cluster_indices = {0, 4, 7, 9}; + } else if (device_name == "S25"){ + cluster_indices = {0, 6}; + } +} + +const std::vector Device::get_cluster_indices() const{ + return this->cluster_indices; +} + +const std::string Device::get_device_name() const{ + return this->device; +} diff --git a/src/hard/device.h b/src/hard/device.h new file mode 100644 index 000000000..20aa39944 --- /dev/null +++ b/src/hard/device.h @@ -0,0 +1,21 @@ +#ifndef DEVICE_H +#define DEVICE_H + +#include +#include +#include + +class Device { +protected: + std::string device; + std::vector cluster_indices; + +public: + Device() = delete; + explicit Device(const std::string& device_name); + + const std::vector get_cluster_indices() const; + const std::string get_device_name() const; +}; + +#endif // DEVICE_HPP diff --git a/src/hard/dvfs.cpp b/src/hard/dvfs.cpp new file mode 100644 index 000000000..49a9b2884 --- /dev/null +++ b/src/hard/dvfs.cpp @@ -0,0 +1,456 @@ +#include "dvfs.h" + +// DVFS -------------------------------------- +const std::map>> DVFS::cpufreq = { + { "S22_Ultra", { + { 0, { 307200, 403200, 518400, 614400, 729600, 844800, 960000, 1075200, 1171200, 1267200, 1363200, 1478400, 1574400, 1689600, 1785600 } }, + { 4, { 633600, 768000, 883200, 998400, 1113600, 1209600, 1324800, 1440000, 1555200, 1651200, 1766400, 1881600, 1996800, 2112000, 2227200, 2342400, 2419200 } }, + { 7, { 806400, 940800, 1056000, 1171200, 1286400, 1401600, 1497600, 1612800, 1728000, 1843200, 1958400, 2054400, 2169600, 2284800, 2400000, 2515200, 2630400, 2726400, 2822400, 2841600 } } + }}, + { "S24", { + { 0, { 400000, 576000, 672000, 768000, 864000, 960000, 1056000, 1152000, 1248000, 1344000, 1440000, 1536000, 1632000, 1728000, 1824000, 1920000, 1959000 } }, + { 4, { 672000, 768000, 864000, 960000, 1056000, 1152000, 1248000, 1344000, 1440000, 1536000, 1632000, 1728000, 1824000, 1920000, 2016000, 2112000, 2208000, 2304000, 2400000, 2496000, 2592000 } }, + { 7, { 672000, 768000, 864000, 960000, 1056000, 1152000, 1248000, 1344000, 1440000, 1536000, 1632000, 1728000, 1824000, 1920000, 2016000, 2112000, 2208000, 2304000, 2400000, 2496000, 2592000, 2688000, 2784000, 2880000, 2900000 } }, + { 9, { 672000, 768000, 864000, 960000, 1056000, 1152000, 1248000, 1344000, 1440000, 1536000, 1632000, 1728000, 1824000, 1920000, 2016000, 2112000, 2208000, 2304000, 2400000, 2496000, 2592000, 2688000, 2784000, 2880000, 2976000, 3072000, 3207000 } } + }}, + { "S25", { + { 0, { 384000, 556800, 748800, 960000, 1152000, 1363200, 1555200, 1785600, 1996800, 2227200, 2400000, 2745600, 2918400, 3072000, 3321600, 3532800 } }, + { 6, { 1017600, 1209600, 1401600, 1689600, 1958400, 2246400, 2438400, 2649600, 2841600, 3072000, 3283200, 3513600, 3840000, 4089600, 4281600, 4473600 } } + }}, + { "Fold4", { + { 0, { 300000, 441600, 556800, 691200, 806400, 940800, 1056000, 1132800, 1228800, 1324800, 1440000, 1555200, 1670400, 1804800, 1920000, 2016000} }, + { 4, { 633600, 768000, 883200, 998400, 1113600, 1209600, 1324800, 1440000, 1555200, 1651200, 1766400, 1881600, 1996800, 2112000, 2227200, 2342400, 2457600, 2572800, 2649600, 2745600 } }, + { 7, { 787200, 921600, 1036800, 1171200, 1286400, 1401600, 1536000, 1651200, 1766400, 1881600, 1996800, 2131200, 2246400, 2361600, 2476800, 2592000, 2707200, 2822400, 2918400, 2995200 } } + }}, + { "Pixel9", { + { 0, { 820000, 955000, 1098000, 1197000, 1328000, 1425000, 1548000, 1696000, 1849000, 1950000 } }, + { 4, { 357000, 578000, 648000, 787000, 910000, 1065000, 1221000, 1328000, 1418000, 1549000, 1795000, 1945000, 2130000, 2245000, 2367000, 2450000, 2600000 } }, + { 7, { 700000, 1164000, 1396000, 1557000, 1745000, 1885000, 1999000, 2147000, 2294000, 2363000, 2499000, 2687000, 2802000, 2914000, 2943000, 2970000, 3015000, 3105000 } } + }} +}; + +const std::map> DVFS::ddrfreq = { + { "S22_Ultra", { 547000, 768000, 1555000, 1708000, 2092000, 2736000, 3196000 } }, + { "S24", { 421000, 676000, 845000, 1014000, 1352000, 1539000, 1716000, 2028000, 2288000, 2730000, 3172000, 3738000, 4206000 } }, + { "S25", { 547000, 1353000, 1555000, 1708000, 2092000, 2736000, 3187000, 3686000, 4224000, 4761000 } }, + { "Fold4", { 547000, 768000, 1555000, 1708000, 2092000, 2736000, 3196000 } }, + { "Pixel9", { 421000, 546000, 676000, 845000, 1014000, 1352000, 1539000, 1716000, 2028000, 2288000, 2730000, 3172000, 3744000 } } +}; + + +const std::map> DVFS::empty_thermal = { + { "S22_Ultra", { "sdr0-pa0", "sdr1-pa0", "pm8350b_tz", "pm8350b-ibat-lvl0", "pm8350b-ibat-lvl1", "pm8350b-bcl-lvl0", "pm8350b-bcl-lvl1", "pm8350b-bcl-lvl2", "socd", "pmr735b_tz"}}, + { "Fold4", { "sdr0-pa0", "sdr1-pa0", "pm8350b_tz", "pm8350b-ibat-lvl0", "pm8350b-ibat-lvl1", "pm8350b-bcl-lvl0", "pm8350b-bcl-lvl1", "pm8350b-bcl-lvl2", "socd", "pmr735b_tz", "qcom,secure-non"}}, + { "S24", {}}, + { "S25", {}}, + { "Pixel9", {}} +}; + + +// consturctor +DVFS::DVFS(const std::string& device_name) : Device(device_name) { output_filename = ""; } +DVFS::~DVFS() { close_fd_cache(); } + + +const std::map>& DVFS::get_cpu_freq() const { + return cpufreq.at(device); +} +const std::vector& DVFS::get_empty_thermal() const { + return empty_thermal.at(device); +} + +const std::vector& DVFS::get_ddr_freq() const { + return ddrfreq.at(device); +} + +std::vector DVFS::get_cpu_freqs_conf(int prime_cpu_index){ + int prime_cluster_id = this->cluster_indices[this->cluster_indices.size()-1]; + int max_prime_cluster_idx = this->get_cpu_freq().at(prime_cluster_id).size()-1; + + // integrity check + if (prime_cpu_index > max_prime_cluster_idx ){ + std::cerr << "[WARNING] Too big prime_cpu_index: " << prime_cpu_index << " > " << max_prime_cluster_idx << std::endl; + } + + + // generate frequency configuration + std::vector freq_conf = {}; + for (auto cluster_idx : this->cluster_indices){ + int max_idx = this->get_cpu_freq().at(cluster_idx).size()-1; + int idx = static_cast( + std::round(((double)prime_cpu_index/(double)max_prime_cluster_idx)*(double)max_idx) + ); + + freq_conf.push_back(idx); + } + + return freq_conf; +} +// ------------------------------------------- + + +// Collector ---------------------------------- +Collector::Collector(const std::string& device_name) : Device(device_name) {} + +// pixel9 +// BIG: thermal/thermal_zone0 +// MID: thermal/thermal_zone1 +const std::map> Collector::thermal_zones_cpu = { + { "Pixel9", { /*BIG*/ "/sys/devices/virtual/thermal/thermal_zone0", /*MID*/ "/sys/devices/virtual/thermal/thermal_zone1" } } +}; + +double Collector::collect_high_temp(){ + if (this->device != "Pixel9") return 0.0; + + std::string command = "su -c \""; + for (auto zone_path : this->thermal_zones_cpu.at(this->device)){ + command += std::string("awk '{print \\$1/1000}' ")+zone_path+std::string("/temp; "); + } + command += "\""; // closing quote + + std::string output = execute_cmd(command.c_str()); + std::vector temps = split_string(output); + + // print high temperature + std::vector temp_vals = {}; + for (auto t_str : temps){ + temp_vals.push_back(std::stod(t_str)); + } + + return std::max_element(temp_vals.begin(), temp_vals.end())[0]; +} + +// ------------------------------------------- + + +int DVFS::open_wr(const std::string& path) { + // open with O_CLOEXEC to prevent FD leak to child processes + int fd = open(path.c_str(), O_WRONLY | O_CLOEXEC); + if (fd < 0) { + fprintf(stderr, "[DVFS] open failed: %s (%s)\n", path.c_str(), strerror(errno)); + } + return fd; +} + +void DVFS::close_fd(int& fd) { + // close fd + if (fd >= 0) { + close(fd); + fd = -1; + } +} + +bool DVFS::try_open_first(const std::vector& candidates, int& out_fd) { + // try open files in candidates sequentially + for (const auto& p : candidates) { + int fd = open_wr(p); + if (fd >= 0) { + out_fd = fd; + return true; + } + } + out_fd = -1; + return false; +} + +int DVFS::write_fd_int(int fd, long long v) { + // write integer status to fd + + if (fd < 0) return -1; + + char buf[64]; + int len = snprintf(buf, sizeof(buf), "%lld\n", v); + if (len <= 0) return -2; + + // sysfs: offset 0 write is safe + (void)lseek(fd, 0, SEEK_SET); + + const char* p = buf; + int left = len; + while (left > 0) { + ssize_t n = write(fd, p, left); + if (n < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "[DVFS] write failed (fd=%d): %s\n", fd, strerror(errno)); + return -3; + } + p += n; + left -= (int)n; + } + return 0; +} + +// 1) FD cache initialization +int DVFS::init_fd_cache() { + std::lock_guard lk(io_mu); + + close_fd_cache_nolock(); // if already opened, close first + + // CPU policy fds + cpu_fds.clear(); + cpu_fds.reserve(cluster_indices.size()); + + for (int idx : cluster_indices) { + CpuPolicyFD p; + p.policy_idx = idx; + + // Pixel9 and S24 have same path structure + const std::string base = "/sys/devices/system/cpu/cpufreq/policy" + std::to_string(idx); + p.max_fd = open_wr(base + "/scaling_max_freq"); + p.min_fd = open_wr(base + "/scaling_min_freq"); + + if (p.max_fd < 0 || p.min_fd < 0) { + fprintf(stderr, "[DVFS] policy%d open incomplete (need root?)\n", idx); + close_fd(p.max_fd); + close_fd(p.min_fd); + // if failure, close all and return error + close_fd_cache(); + fd_ready = false; + return -1; + } + + cpu_fds.push_back(p); + } + + // MIF(devfreq) fds (RAM) + if (get_device_name() == "S25") { + // S25 uses bus_dcvs RAM voters. Keep the chmod-dependent prime-latfloor nodes + // excluded, but cache the remaining writable nodes for direct FD-based control. + s25_ram_fds.ddr_boost_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/boost_freq"); + s25_ram_fds.ddrqos_boost_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDRQOS/boost_freq"); + + s25_ram_fds.ddr_gold_min_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:gold/min_freq"); + s25_ram_fds.ddr_gold_max_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:gold/max_freq"); + s25_ram_fds.ddr_gold_compute_min_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:gold-compute/min_freq"); + s25_ram_fds.ddr_gold_compute_max_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:gold-compute/max_freq"); + s25_ram_fds.ddr_prime_min_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:prime/min_freq"); + s25_ram_fds.ddr_prime_max_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDR/soc:qcom,memlat:ddr:prime/max_freq"); + + s25_ram_fds.ddrqos_gold_min_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDRQOS/soc:qcom,memlat:ddrqos:gold/min_freq"); + s25_ram_fds.ddrqos_gold_max_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDRQOS/soc:qcom,memlat:ddrqos:gold/max_freq"); + s25_ram_fds.ddrqos_prime_min_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDRQOS/soc:qcom,memlat:ddrqos:prime/min_freq"); + s25_ram_fds.ddrqos_prime_max_fd = open_wr("/sys/devices/system/cpu/bus_dcvs/DDRQOS/soc:qcom,memlat:ddrqos:prime/max_freq"); + + if (s25_ram_fds.ddr_boost_fd < 0 || + s25_ram_fds.ddrqos_boost_fd < 0 || + s25_ram_fds.ddr_gold_min_fd < 0 || + s25_ram_fds.ddr_gold_max_fd < 0 || + s25_ram_fds.ddr_gold_compute_min_fd < 0 || + s25_ram_fds.ddr_gold_compute_max_fd < 0 || + s25_ram_fds.ddr_prime_min_fd < 0 || + s25_ram_fds.ddr_prime_max_fd < 0 || + s25_ram_fds.ddrqos_gold_min_fd < 0 || + s25_ram_fds.ddrqos_gold_max_fd < 0 || + s25_ram_fds.ddrqos_prime_min_fd < 0 || + s25_ram_fds.ddrqos_prime_max_fd < 0) { + fprintf(stderr, "[DVFS] S25 RAM voter open failed (need root?)\n"); + close_fd_cache(); + fd_ready = false; + return -2; + } + fd_ready = true; + return 0; + } + // MIF(devfreq) fds (RAM) + // Pixel 9 and S24 have same base path + mif_fds.base = "/sys/devices/platform/17000010.devfreq_mif/devfreq/17000010.devfreq_mif"; + { + // Depending on device and kernel, the min/max path differs + std::vector min_candidates = { + mif_fds.base + "/scaling_devfreq_min", // S24 + mif_fds.base + "/min_freq", // Pixel9 + mif_fds.base + "/scaling_min_freq" + }; + + std::vector max_candidates; + if (get_device_name() == "Pixel9") { + max_candidates = { + mif_fds.base + "/max_freq", // Pixel9 preferred + mif_fds.base + "/scaling_devfreq_max" + }; + } else { + max_candidates = { + mif_fds.base + "/scaling_devfreq_max", // S24 preferred + mif_fds.base + "/max_freq" + }; + } + + if (!try_open_first(min_candidates, mif_fds.min_fd)) { + fprintf(stderr, "[DVFS] MIF min open failed (need root? path mismatch)\n"); + close_fd_cache(); + fd_ready = false; + return -2; + } + if (!try_open_first(max_candidates, mif_fds.max_fd)) { + fprintf(stderr, "[DVFS] MIF max open failed (need root? path mismatch)\n"); + close_fd_cache(); + fd_ready = false; + return -3; + } + } + + fd_ready = true; + return 0; +} + +// 2) FD cache cleanup +void DVFS::close_fd_cache() { + std::lock_guard lk(io_mu); + close_fd_cache_nolock(); +} + +void DVFS::close_fd_cache_nolock() { + // close all cached fds with no lock + // assume io_mu is already locked + // to avoid deadlock + for (auto& p : cpu_fds) { + close_fd(p.max_fd); + close_fd(p.min_fd); + } + cpu_fds.clear(); + + close_fd(mif_fds.min_fd); + close_fd(mif_fds.max_fd); + close_fd(s25_ram_fds.ddr_boost_fd); + close_fd(s25_ram_fds.ddrqos_boost_fd); + close_fd(s25_ram_fds.ddr_gold_min_fd); + close_fd(s25_ram_fds.ddr_gold_max_fd); + close_fd(s25_ram_fds.ddr_gold_compute_min_fd); + close_fd(s25_ram_fds.ddr_gold_compute_max_fd); + close_fd(s25_ram_fds.ddr_prime_min_fd); + close_fd(s25_ram_fds.ddr_prime_max_fd); + close_fd(s25_ram_fds.ddrqos_gold_min_fd); + close_fd(s25_ram_fds.ddrqos_gold_max_fd); + close_fd(s25_ram_fds.ddrqos_prime_min_fd); + close_fd(s25_ram_fds.ddrqos_prime_max_fd); + + fd_ready = false; +} + +// 3) set/unset: directly write if FD cache is ready +int DVFS::set_cpu_freq(const std::vector& freq_indices) { + if ((int)cluster_indices.size() != (int)freq_indices.size()) return 1; + + std::lock_guard lk(io_mu); + + if (!fd_ready) { + fprintf(stderr, "[DVFS] fd cache not ready. call init_fd_cache() first.\n"); + return 2; + } + + // max first, min last (protect min > max being set) + for (int i = 0; i < (int)cluster_indices.size(); ++i) { + int policy = cluster_indices[i]; + int freq_idx = freq_indices[i]; + + const auto& table = cpufreq.at(device).at(policy); + if (freq_idx < 0 || freq_idx >= (int)table.size()) return 3; + + int clk = table[freq_idx]; + + // search corresponding policy fd (if sequence is identical, cpu_fds[i] can be used directly) + // safe policy match + CpuPolicyFD* fdp = nullptr; + for (auto& p : cpu_fds) if (p.policy_idx == policy) { fdp = &p; break; } + if (!fdp) return 4; + + if (write_fd_int(fdp->max_fd, clk) != 0) return 5; + if (write_fd_int(fdp->min_fd, clk) != 0) return 6; + } + return 0; +} + +int DVFS::unset_cpu_freq() { + // unset to default (min: lowest, max: highest) + + std::lock_guard lk(io_mu); + + if (!fd_ready) { + fprintf(stderr, "[DVFS] fd cache not ready. call init_fd_cache() first.\n"); + return 2; + } + + for (int policy : cluster_indices) { + const auto& table = cpufreq.at(device).at(policy); + int min_clk = table.front(); + int max_clk = table.back(); + + CpuPolicyFD* fdp = nullptr; + for (auto& p : cpu_fds) if (p.policy_idx == policy) { fdp = &p; break; } + if (!fdp) return 4; + + if (write_fd_int(fdp->max_fd, max_clk) != 0) return 5; + if (write_fd_int(fdp->min_fd, min_clk) != 0) return 6; + } + return 0; +} + +int DVFS::set_ram_freq(const int freq_idx) { + std::lock_guard lk(io_mu); + + if (!fd_ready) { + fprintf(stderr, "[DVFS] fd cache not ready. call init_fd_cache() first.\n"); + return 2; + } + + const auto& table = get_ddr_freq(); + if (freq_idx < 0 || freq_idx >= (int)table.size()) return 1; + + int clk = table[freq_idx]; + + if (this->get_device_name() == "S25") { + if (write_fd_int(s25_ram_fds.ddr_boost_fd, clk) != 0) return 3; + if (write_fd_int(s25_ram_fds.ddrqos_boost_fd, 0) != 0) return 4; + + if (write_fd_int(s25_ram_fds.ddr_gold_min_fd, clk) != 0) return 5; + if (write_fd_int(s25_ram_fds.ddr_gold_max_fd, clk) != 0) return 6; + if (write_fd_int(s25_ram_fds.ddr_gold_compute_min_fd, clk) != 0) return 7; + if (write_fd_int(s25_ram_fds.ddr_gold_compute_max_fd, clk) != 0) return 8; + if (write_fd_int(s25_ram_fds.ddr_prime_min_fd, clk) != 0) return 9; + if (write_fd_int(s25_ram_fds.ddr_prime_max_fd, clk) != 0) return 10; + + if (write_fd_int(s25_ram_fds.ddrqos_gold_min_fd, 1) != 0) return 11; + if (write_fd_int(s25_ram_fds.ddrqos_gold_max_fd, 1) != 0) return 12; + if (write_fd_int(s25_ram_fds.ddrqos_prime_min_fd, 1) != 0) return 13; + if (write_fd_int(s25_ram_fds.ddrqos_prime_max_fd, 1) != 0) return 14; + return 0; + } + // max first, min last (policy-dependent, but this form is generally safe) + if (write_fd_int(mif_fds.max_fd, clk) != 0) return 3; + if (write_fd_int(mif_fds.min_fd, clk) != 0) return 4; + return 0; +} + +int DVFS::unset_ram_freq() { + std::lock_guard lk(io_mu); + + if (!fd_ready) { + fprintf(stderr, "[DVFS] fd cache not ready. call init_fd_cache() first.\n"); + return 2; + } + + const auto& table = get_ddr_freq(); + int min_clk = table.front(); + int max_clk = table.back(); + + if (this->get_device_name() == "S25") { + if (write_fd_int(s25_ram_fds.ddr_boost_fd, min_clk) != 0) return 3; + if (write_fd_int(s25_ram_fds.ddrqos_boost_fd, 0) != 0) return 4; + + if (write_fd_int(s25_ram_fds.ddr_gold_min_fd, min_clk) != 0) return 5; + if (write_fd_int(s25_ram_fds.ddr_gold_max_fd, max_clk) != 0) return 6; + if (write_fd_int(s25_ram_fds.ddr_gold_compute_min_fd, min_clk) != 0) return 7; + if (write_fd_int(s25_ram_fds.ddr_gold_compute_max_fd, max_clk) != 0) return 8; + if (write_fd_int(s25_ram_fds.ddr_prime_min_fd, min_clk) != 0) return 9; + if (write_fd_int(s25_ram_fds.ddr_prime_max_fd, max_clk) != 0) return 10; + + if (write_fd_int(s25_ram_fds.ddrqos_gold_min_fd, 0) != 0) return 11; + if (write_fd_int(s25_ram_fds.ddrqos_gold_max_fd, 1) != 0) return 12; + if (write_fd_int(s25_ram_fds.ddrqos_prime_min_fd, 0) != 0) return 13; + if (write_fd_int(s25_ram_fds.ddrqos_prime_max_fd, 1) != 0) return 14; + return 0; + } + if (write_fd_int(mif_fds.max_fd, max_clk) != 0) return 3; + if (write_fd_int(mif_fds.min_fd, min_clk) != 0) return 4; + return 0; +} diff --git a/src/hard/dvfs.h b/src/hard/dvfs.h new file mode 100644 index 000000000..4a5bae110 --- /dev/null +++ b/src/hard/dvfs.h @@ -0,0 +1,133 @@ +#ifndef DVFS_H +#define DVFS_H + +#include "device.h" +#include "utils.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +class Collector; + +class Collector : public Device { +private: + // pixel9 + // BIG: thermal/thermal_zone0 + // MID: thermal/thermal_zone1 + static const std::map> thermal_zones_cpu; +public: + explicit Collector(const std::string& device_name); + double collect_high_temp(); + +}; + + +/* ** Example of DVFS class ** + +DVFS dvfs("Pixel9"); +if (dvfs.init_fd_cache() != 0) { + fprintf(stderr, "FD cache initialization failed. Are you root or authorized?\n"); +} + +... (skip) ... + +std::vector freq_config = dvfs.get_cpu_freqs_conf(prime_cpu_index); +dvfs.set_cpu_freq(freq_config); +dvfs.set_ram_freq(ram_freq_index); +... (skip) ... +dvfs.unset_cpu_freq(); +dvfs.unset_ram_freq(); + +*/ +class DVFS : public Device { +private: + static const std::map>> cpufreq; + static const std::map> ddrfreq; + static const std::map> empty_thermal; + +private: + // ---- FD cache structure ---- + struct CpuPolicyFD { + int policy_idx = -1; + int min_fd = -1; // scaling_min_freq + int max_fd = -1; // scaling_max_freq + }; + + struct MifFD { + int min_fd = -1; // scaling_devfreq_min (or min_freq) + int max_fd = -1; // scaling_devfreq_max (or max_freq) + std::string base; + }; + + struct S25RamFD { + int ddr_boost_fd = -1; + int ddrqos_boost_fd = -1; + + int ddr_gold_min_fd = -1; + int ddr_gold_max_fd = -1; + int ddr_gold_compute_min_fd = -1; + int ddr_gold_compute_max_fd = -1; + int ddr_prime_min_fd = -1; + int ddr_prime_max_fd = -1; + + int ddrqos_gold_min_fd = -1; + int ddrqos_gold_max_fd = -1; + int ddrqos_prime_min_fd = -1; + int ddrqos_prime_max_fd = -1; + }; + + std::vector cpu_fds; + MifFD mif_fds; + S25RamFD s25_ram_fds; + bool fd_ready = false; + std::mutex io_mu; // mutex lock guard for fd cache I/O + +public: + std::string output_filename; + const std::chrono::system_clock::time_point zero_start_point{std::chrono::system_clock::time_point::duration::zero()}; // not changed + std::chrono::system_clock::time_point control_start_point{std::chrono::system_clock::time_point::duration::zero()}; + +public: + DVFS(const std::string& device_name); + ~DVFS(); + + const std::map>& get_cpu_freq() const; + const std::vector& get_ddr_freq() const; + const std::vector& get_empty_thermal() const; + + int set_cpu_freq(const std::vector&); + int unset_cpu_freq(); + int set_ram_freq(const int freq_idx); + int unset_ram_freq(); + + std::vector get_cpu_freqs_conf(int prime_cpu_index); + + Collector get_collector() { return Collector(this->get_device_name()); } + + // FD cache + int init_fd_cache(); // sysfs open + void close_fd_cache(); // sysfs close + bool fd_cache_enabled() const { return fd_ready; } + +private: + // internal helper + static int open_wr(const std::string& path); + static int write_fd_int(int fd, long long v); + static void close_fd(int& fd); + static bool try_open_first(const std::vector& candidates, int& out_fd); + void close_fd_cache_nolock(); +}; + +#endif //DVFS_H diff --git a/src/hard/record.cpp b/src/hard/record.cpp new file mode 100644 index 000000000..096b8bebe --- /dev/null +++ b/src/hard/record.cpp @@ -0,0 +1,318 @@ +#include "record.h" +#include + +// test function +void get_cpu_info() { + std::string command = "su -c \""; //prefix + + // command to get cpu freq + command += "awk '{print \\$1/1000}' /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq; "; + command += "awk '{print \\$1/1000}' /sys/devices/system/cpu/cpu4/cpufreq/scaling_cur_freq; "; + command += "\""; // postfix + + // only execution + system(command.c_str()); + //std::string output = execute_cmd(command.c_str()); + //std::cout << execute_cmd(command.c_str())[0] << std::endl; +} + +const std::string get_records_names(const DVFS& dvfs) { + std::string names = "Time,"; + + // thermal info + std::string command = "su -c \"cat /sys/devices/virtual/thermal/thermal_zone*/type\""; + std::string temp_record = execute_cmd(command.c_str()); + std::replace(temp_record.begin(), temp_record.end(), '\n', ','); + names += temp_record; + + // gpu info + names += "gpu_min_clock,gpu_max_clock,"; + + // cpu info + for(const auto index : dvfs.get_cluster_indices()){ + names += std::string("cpu") + std::to_string(index) + std::string("_max_freq,cpu") + std::to_string(index) + "_cur_freq,"; + } + + // mem info + command = "su -c \"awk '{print \\$1}' /proc/meminfo\""; + temp_record = execute_cmd(command.c_str()); + std::replace(temp_record.begin(), temp_record.end(), ':', '\0'); + std::replace(temp_record.begin(), temp_record.end(), '\n', ','); + names += temp_record; + + // power + if (dvfs.get_device_name() == "Pixel9") names += "current_now,voltage_now,"; + else names += "power_now,current_now,voltage_now,"; + + // RAM clock info + names += "scaling_devfreq_max,scaling_devfreq_min,cur_freq,"; + + // remove emptyThermal + for (std::string empty : dvfs.get_empty_thermal()){ + if (empty == "qcom,secure-non"){ + std::size_t p = 0; + while( (p = names.find(empty, p)) != std::string::npos ){ + names.replace(p, empty.length(), "secure-non"); + } + continue; + } + std::string temp = empty + ","; + std::size_t pos = 0; + while( (pos = names.find(temp, pos)) != std::string::npos){ + names.replace(pos, temp.length(), ""); // string replace + } + } + + // [ADD] LLCC clock info (Only for S25) + if (dvfs.get_device_name() == "S25") { + names += "llcc_prime_cur_freq,llcc_gold_cur_freq,llcc_gold_compute_cur_freq,llcc_cur_freq,bwmon_llcc_gold_cur_freq,bwmon_llcc_prime_cur_freq,"; + } + + return names; +} + +/* + * GET HARD RECORDS function + * - args + * - cluster_indices: an integer vector or array to contain cluster indices (ex. {0,4,7}) + * - task + * - Get hard records such as thermal, power, etc. + * - return + * - A string vector to contain outputs + * */ +std::vector get_hard_records(const DVFS& dvfs) { + std::vector cluster_indices = dvfs.get_cluster_indices(); + std::string device_name = dvfs.get_device_name(); + + std::string command = "su -c \""; // prefix + + // thermal info + command += "awk '{print \\$1/1000}' /sys/devices/virtual/thermal/thermal_zone*/temp; "; + + // GPU clock info + if (device_name == "Pixel9"){ + command += "awk '{print \\$1}' /sys/devices/platform/1f000000.mali/scaling_min_freq; awk '{print \\$1}' /sys/devices/platform/1f000000.mali/scaling_max_freq; "; //gpu clock + } else { // S24 + command += "awk '{print \\$1}' /sys/kernel/gpu/gpu_min_clock; awk '{print \\$1}' /sys/kernel/gpu/gpu_max_clock; "; + } + + // CPU clock info + for (std::size_t i=0; i get_hard_records_wo_systime(const DVFS& dvfs){ + std::vector cluster_indices = dvfs.get_cluster_indices(); + std::string device_name = dvfs.get_device_name(); + + std::string command = "su -c \""; // prefix + + // thermal info + command += "awk '{print \\$1/1000}' /sys/devices/virtual/thermal/thermal_zone*/temp; "; + + // GPU clock info + if (device_name == "Pixel9"){ + command += "awk '{print \\$1}' /sys/devices/platform/1f000000.mali/scaling_min_freq; awk '{print \\$1}' /sys/devices/platform/1f000000.mali/scaling_max_freq; "; //gpu clock + } else { // S24 + command += "awk '{print \\$1}' /sys/kernel/gpu/gpu_min_clock; awk '{print \\$1}' /sys/kernel/gpu/gpu_max_clock; "; + } + + // CPU clock info + for (std::size_t i=0; i& data, std::string output){ + + // open file append mode + std::ofstream file(output, std::ios::app); + + // check file open + if (!file){ + std::cerr << "failed to open file: " << HARD_RECORD_FILE << std::endl; + return; + } + + // wrtie file + for (const auto v : data){ + file << v << ","; + } + file << "\n"; + + // close file + file.close(); +} + +void write_file(const std::string& data, std::string output){ + // open file append mode + std::ofstream file(output, std::ios::app); + + // check file open + if (!file){ + std::cerr << "failed to open file: " << HARD_RECORD_FILE << std::endl; + return; + } + + // wrtie file + file << data << "\n"; + + // close file + file.close(); +} + + +/* + * + * ### This function should be called by background process! + * ### sigterm should be true after experiment completion + * + * */ +void record_hard(std::atomic& sigterm, const DVFS& dvfs){ + pin_current({2}); // generally silver core on mobile + sigterm = false; + std::string filename = dvfs.output_filename; + + + // insert hard names + write_file(get_records_names(dvfs), filename); + + + int test_index = 0; + std::vector records; + auto start_sys_time = std::chrono::system_clock::now(); + if (dvfs.control_start_point != dvfs.zero_start_point) { + // if control_start_point time is given, + // now start_sys_time is replaced with the given control_start_point + start_sys_time = dvfs.control_start_point; + } + + do{ + // get records + records = get_hard_records(dvfs); + auto now = std::chrono::system_clock::now(); + auto sys_time = std::chrono::duration_cast(now-start_sys_time).count(); // ms base + records.insert(records.begin(), std::to_string(sys_time)); // insert systime into firstrecord element + + // File write record + write_file(records, filename); + // wait + //std::this_thread::sleep_for(std::chrono::milliseconds(170)); + +// tester code: start +// test_index++; +// if (test_index == 3) sigterm = true; +// tester code: end + + }while(sigterm != true); +} diff --git a/src/hard/record.h b/src/hard/record.h new file mode 100644 index 000000000..efb94abcf --- /dev/null +++ b/src/hard/record.h @@ -0,0 +1,39 @@ + +#ifndef RECORD +#define RECORD + +#include "dvfs.h" +#include "utils.h" +#include "affinity.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HARD_RECORD_FILE "./data/hard_info_termux.txt" +#define INFER_RECORD_FILE "./data/infer_info.csv" +#define TIME_T std::chrono::system_clock::time_point + +//test +void get_cpu_info(); + +// get function +const std::string get_records_names(const DVFS& dvfs); +std::vector get_hard_records(const DVFS& dvfs); +std::vector get_hard_records_wo_systime(const DVFS& dvfs); + +// write function +void write_file(const std::vector& data, std::string output); +void write_file(const std::string& data, std::string output); +void record_hard(std::atomic& sigterm, const DVFS& dvfs); + +#endif diff --git a/src/hard/utils.cpp b/src/hard/utils.cpp new file mode 100644 index 000000000..51ed63284 --- /dev/null +++ b/src/hard/utils.cpp @@ -0,0 +1,165 @@ +#include "utils.h" + +bool is_csv_file(const std::string & filename) { + fs::path p(filename); + if (fs::exists(p) && p.extension() == ".csv") { + return true; + } + return false; +} +bool is_json_file(const std::string & filename) { + fs::path p(filename); + if (fs::exists(p) && p.extension() == ".json") { + return true; + } + return false; +} +static std::vector parseCSVLine(const std::string& line) { + std::vector values; + std::string current; + bool insideQuotes = false; + + for (char ch : line) { + if (ch == '"') { + insideQuotes = !insideQuotes; + } else if (ch == ',' && !insideQuotes) { + values.push_back(current); + current.clear(); + } else { + current += ch; + } + } + values.push_back(current); // last field + + return values; +} + +std::vector> readCSV(const std::string& filename) { + std::vector> result; + std::ifstream file(filename); + + if (!file.is_open()) { + std::cerr << "cannot open file: " << filename << std::endl; + return result; + } + + std::string line; std::size_t i = 0; + while (std::getline(file, line)) { + result.push_back(parseCSVLine(line)); + } + + file.close(); + return result; +} + +std::vector readJSON(const std::string& filename){ + // A parsing function for "questions.json" with very simple way + // The following is JSON file type: + // { + // "questions": [ + // "the first content of question", + // "the second content of question", + // "the third content of question" + // ] + // } + std::ifstream file(filename); + std::vector qs; + try { + json jsonData; file >> jsonData; // JSON parsing + + if (jsonData.contains("questions") && jsonData["questions"].is_array()) { + for (const auto& item : jsonData["questions"]) { + if (item.is_string()) { qs.push_back(item.get()); } + } + } else { std::cerr << "Invalid JSON format: 'data' key missing or not an array\n"; } + } catch (const std::exception &e) { + std::cerr << "JSON parsing error: " << e.what() << "\n"; + } + return qs; +} + +std::vector loadQuestions(const std::string &filename) { + std::vector questions; + // csv case + if (is_csv_file(filename)) { + // If CSV data is found, extract the second column as questions + for (const auto& row : readCSV(filename)) { + if (!row.empty()) questions.push_back(row[1]); + } + return questions; + } + + // json case + if (is_json_file(filename)) return readJSON(filename); + + // no supported + std::cerr << "Unsupported file format. Did not read: " << filename << "\n"; + + return questions; +} + +std::string joinPaths(const std::string& path1, const std::string& path2) { + if (path1.empty()) return path2; + if (path2.empty()) return path1; + + char lastChar = path1[path1.length() - 1]; + char firstChar = path2[0]; + + if (lastChar == '/' && firstChar == '/') { + return path1 + path2.substr(1); + } else if (lastChar != '/' && firstChar != '/') { + return path1 + "/" + path2; + } else { + return path1 + path2; + } +} + +std::string replace(std::string origin, std::string target, std::string destination) { + size_t pos = 0; + while ((pos = origin.find(target, pos)) != std::string::npos) { + origin.replace(pos, target.length(), destination); + pos += destination.length(); + } + return origin; +} + +std::vector split_string(const std::string & str){ + //initialization + std::vector result; + // conversion to stream string + std::istringstream iss(str); + std::string value; // splitted value + + while (iss >> value){ + result.push_back(value); + } + + return result; +} + +std::string execute_cmd(const char* cmd) { + // command execution + FILE* pipe = popen(cmd, "r"); + + // check pipe open + if (!pipe) { fprintf(stderr, "failed to pipe open (record.h)\n"); return ""; } + + // get output from buffer + std::ostringstream result; + char buff[8192]; + while (fgets(buff, sizeof(buff), pipe) != nullptr) { + result << buff; + } + + pclose(pipe); + return result.str(); +} + +std::string apply_sudo_and_get(std::string command) { + std::string cmd = "su -c \""; // prefix + if (command != "") cmd += command; + else cmd += "awk '{print \\$1/1000}' /sys/devices/system/cpu/cpu7/cpufreq/scaling_cur_freq"; // command + cmd += "\""; // postfix + + return cmd; +} \ No newline at end of file diff --git a/src/hard/utils.h b/src/hard/utils.h new file mode 100644 index 000000000..db55eae36 --- /dev/null +++ b/src/hard/utils.h @@ -0,0 +1,36 @@ +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include +#include + +// #include "nlohmann/json.hpp" +#include "nlohmann/json.hpp" + +// TODO: move to files.h/.cpp +// File utils +namespace fs = std::filesystem; +using json = nlohmann::json; +bool is_csv_file(const std::string & filename); +bool is_json_file(const std::string & filename); +static std::vector parseCSVLine(const std::string& line); +std::vector> readCSV(const std::string& filename); +std::vector readJSON(const std::string& filename); +std::vector loadQuestions(const std::string &filename); +std::string joinPaths(const std::string& path1, const std::string& path2); + +// string utils +std::vector split_string(const std::string & str); +std::string replace(std::string origin, std::string target, std::string destination); + +// internal static functions +std::string execute_cmd(const char* cmd); + +// throttling detection support +std::string apply_sudo_and_get(std::string command); + +#endif // UTILS_H diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 95b207e9e..cdf17af28 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -8,11 +8,14 @@ #include "llama-mmap.h" #include "llama-model.h" +#include #include #include #include +#include #include #include +#include // // llama_context @@ -1073,6 +1076,18 @@ void llama_context::set_adapter_lora( sched_need_reserve = true; } +void llama_context::set_ignite_params( + const llama_igparams * cfg) { + LLAMA_LOG_DEBUG("%s: call\n", __func__); + igparams = *cfg; + lp_enable = igparams.layer_pause > 0; + ggml_backend_sched_profile_set_enabled(igparams.backend_compute_profile); +} + +struct llama_igparams * llama_context::get_ignite_params() { + return &igparams; +} + bool llama_context::rm_adapter_lora( llama_adapter_lora * adapter) { LLAMA_LOG_DEBUG("%s: adapter = %p\n", __func__, (void *) adapter); @@ -1121,12 +1136,21 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll return nullptr; } + lp_is_prefill = (ubatch.n_tokens > 1); + auto * res = gf_res_prev.get(); auto * gf = res->get_gf(); // the new graph parameters // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters const auto gparams = graph_params(res, ubatch, mctx, gtype); + if (seen_attn_out) { + lp_mha_key = lp_mha_key_t::attn_out; + } else if (seen_kqv_out) { + lp_mha_key = lp_mha_key_t::kqv_out; + } else { + lp_mha_key = lp_mha_key_t::none; + } if (!graph_reuse_disable && res->can_reuse(gparams)) { //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__); @@ -1138,11 +1162,14 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll ggml_backend_sched_reset(sched.get()); ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); - //const auto t_start_us = ggml_time_us(); + const bool profile_backend_compute = igparams.backend_compute_profile; + const int64_t t_build_us = profile_backend_compute ? ggml_time_us() : 0; gf = model.build_graph(gparams); - //LLAMA_LOG_INFO("graph build time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0); + if (profile_backend_compute) { + ggml_backend_sched_profile_add_build_ms((ggml_time_us() - t_build_us) / 1000.0); + } if (!gf) { LLAMA_LOG_ERROR("%s: failed to initialize graph\n", __func__); @@ -1150,7 +1177,12 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll return nullptr; } - if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) { + const int64_t t_alloc_us = profile_backend_compute ? ggml_time_us() : 0; + const bool alloc_ok = ggml_backend_sched_alloc_graph(sched.get(), gf); + if (profile_backend_compute) { + ggml_backend_sched_profile_add_build_ms((ggml_time_us() - t_alloc_us) / 1000.0); + } + if (!alloc_ok) { LLAMA_LOG_ERROR("%s: failed to allocate graph\n", __func__); ret = GGML_STATUS_ALLOC_FAILED; return nullptr; @@ -1159,11 +1191,14 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll // set the input data for the input tensors { - //const auto t_start_us = ggml_time_us(); + const bool profile_backend_compute = igparams.backend_compute_profile; + const int64_t t_inputs_us = profile_backend_compute ? ggml_time_us() : 0; res->set_inputs(&ubatch); - //LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0); + if (profile_backend_compute) { + ggml_backend_sched_profile_add_build_ms((ggml_time_us() - t_inputs_us) / 1000.0); + } } const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1); @@ -2131,6 +2166,17 @@ ggml_status llama_context::graph_compute( set_n_threads_fn.second(set_n_threads_fn.first, n_threads); } + if (igparams.is_ignite_active && lp_enable && batched) { + ggml_backend_sched_set_eval_callback(sched.get(), lp_eval_callback, this); + } else { + ggml_backend_sched_set_eval_callback(sched.get(), nullptr, nullptr); + } + + ggml_backend_sched_profile_set_enabled(igparams.backend_compute_profile); + if (igparams.backend_compute_profile) { + ggml_backend_sched_profile_set_phase(batched ? GGML_BACKEND_SCHED_PROFILE_PREFILL : GGML_BACKEND_SCHED_PROFILE_DECODE); + } + auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); if (status != GGML_STATUS_SUCCESS) { LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); @@ -2141,6 +2187,50 @@ ggml_status llama_context::graph_compute( return status; } +bool llama_context::lp_eval_callback(struct ggml_tensor * t, bool ask, void * user_data) { + auto * ctx = static_cast(user_data); + + if (!ctx->lp_enable || !ctx->lp_is_prefill) { + return ask ? false : true; + } + + const char * n = ggml_get_name(t); + if (!n || !n[0]) { + return ask ? false : true; + } + + const bool is_mha = + (strncmp(n, "attn_out", 8) == 0 && ctx->lp_mha_key == lp_mha_key_t::attn_out) || + (strncmp(n, "kqv_out", 7) == 0 && ctx->lp_mha_key == lp_mha_key_t::kqv_out); + const bool is_ffn = + (strncmp(n, "ffn_out", 7) == 0 || strncmp(n, "ffn_mlp", 7) == 0); + + if (ask) { + return is_mha || is_ffn; + } + + if (strncmp(n, "attn_out", 8) == 0 && ctx->lp_mha_key == lp_mha_key_t::attn_out) { + if (ctx->igparams.ignite_verbose) { + std::cout << std::flush << "igparams.layer_pause << ">"; + } + std::this_thread::sleep_for(std::chrono::milliseconds(ctx->igparams.layer_pause)); + } else if (strncmp(n, "kqv_out", 7) == 0 && ctx->lp_mha_key == lp_mha_key_t::kqv_out) { + if (ctx->igparams.ignite_verbose) { + std::cout << std::flush << "igparams.layer_pause << ">"; + } + std::this_thread::sleep_for(std::chrono::milliseconds(ctx->igparams.layer_pause)); + } + + if (strncmp(n, "ffn_out", 7) == 0 || strncmp(n, "ffn_mlp", 7) == 0) { + if (ctx->igparams.ignite_verbose) { + std::cout << std::flush << "igparams.layer_pause << ">"; + } + std::this_thread::sleep_for(std::chrono::milliseconds(ctx->igparams.layer_pause)); + } + + return is_mha || is_ffn; +} + llm_graph_cb llama_context::graph_get_cb() const { return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) { if (il >= 0) { @@ -2149,6 +2239,13 @@ llm_graph_cb llama_context::graph_get_cb() const { ggml_set_name(cur, name); } + if (strcmp(name, "attn_out") == 0) { + seen_attn_out = true; + } + if (strcmp(name, "kqv_out") == 0) { + seen_kqv_out = true; + } + // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends // FIXME: fix in ggml_backend_sched const bool full_offload = model.n_gpu_layers() > model.hparams.n_layer; diff --git a/src/llama-context.h b/src/llama-context.h index 8e71cdd1d..fec558a22 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -4,6 +4,7 @@ #include "llama-cparams.h" #include "llama-graph.h" #include "llama-adapter.h" +#include "llama-ignite.h" #include "ggml-cpp.h" #include "ggml-opt.h" @@ -108,6 +109,11 @@ struct llama_context { llama_adapter_lora * adapter, float scale); + void set_ignite_params( + const llama_igparams * cfg); + + struct llama_igparams * get_ignite_params(); + bool rm_adapter_lora( llama_adapter_lora * adapter); @@ -229,6 +235,8 @@ struct llama_context { // can reuse the llm_graph_result instance of the context (for example to update a memory module) llm_graph_result * get_gf_res_reserve() const; + static bool lp_eval_callback(struct ggml_tensor * t, bool ask, void * user_data); + // returns the result of ggml_backend_sched_graph_compute_async execution ggml_status graph_compute(ggml_cgraph * gf, bool batched); @@ -367,4 +375,14 @@ struct llama_context { mutable int32_t n_eval = 0; // number of eval calls mutable int32_t n_reused = 0; // number of times the previous graph was reused + + llama_igparams igparams = {}; + +private: + enum class lp_mha_key_t { none, attn_out, kqv_out }; + bool lp_enable = false; + bool lp_is_prefill = false; + mutable bool seen_attn_out = false; + mutable bool seen_kqv_out = false; + mutable lp_mha_key_t lp_mha_key = lp_mha_key_t::none; }; diff --git a/src/llama-ignite.cpp b/src/llama-ignite.cpp new file mode 100644 index 000000000..18e75351d --- /dev/null +++ b/src/llama-ignite.cpp @@ -0,0 +1,176 @@ +#include "llama-ignite.h" + +#include "llama-context.h" + +#include "hard/utils.h" + +#include +#include + +void llama_ignite_set_active(struct llama_context * ctx, bool active) { + if (!ctx) { + return; + } + + auto * ig = ctx->get_ignite_params(); + if (ig == nullptr) { + return; + } + + ig->is_ignite_active = active; +} + +bool llama_ignite_get_active(struct llama_context * ctx) { + if (!ctx) { + return false; + } + + auto * ig = ctx->get_ignite_params(); + return ig != nullptr ? ig->is_ignite_active : false; +} + +void llama_ignite_set_layer_pause(struct llama_context * ctx, uint16_t ms) { + if (!ctx) { + return; + } + + auto * ig = ctx->get_ignite_params(); + if (ig == nullptr) { + return; + } + + ig->layer_pause = ms; + ctx->set_ignite_params(ig); +} + +uint16_t llama_ignite_get_layer_pause(struct llama_context * ctx) { + if (!ctx) { + return 0; + } + + auto * ig = ctx->get_ignite_params(); + return ig != nullptr ? ig->layer_pause : 0; +} + +bool init_ignite_params(struct llama_context * ctx, llama_igparams * igparams) { + if (!ctx || !igparams) { + return false; + } + + ctx->set_ignite_params(igparams); + return true; +} + +struct llama_igparams * get_ignite_params(struct llama_context * ctx) { + if (!ctx) { + return nullptr; + } + + return ctx->get_ignite_params(); +} + +bool init_ignite_filename(struct llama_context * ctx) { + if (!ctx) { + return false; + } + + struct llama_igparams * ig = ctx->get_ignite_params(); + if (ig == nullptr) { + return false; + } + + const bool fixed_config = (ig->cpu_clk_idx_p == ig->cpu_clk_idx_d) && (ig->ram_clk_idx_p == ig->ram_clk_idx_d); + const bool tp = (ig->token_pause > 0); + const bool pp = (ig->phase_pause > 0); + const bool lp = (ig->layer_pause > 0); + const bool qi = (ig->query_interval > 0); + char mode = 0b00000000; + + mode |= (!fixed_config) ? (1 << 0) : 0; + mode |= pp ? (1 << 1) : 0; + mode |= lp ? (1 << 2) : 0; + mode |= tp ? (1 << 3) : 0; + mode |= qi ? (1 << 4) : 0; + + std::string output_hard; + std::string output_infer; + + switch (mode) { + case 0: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_infer.txt"); + break; + case 1: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_infer.txt"); + break; + case 2: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_pp_" + std::to_string(ig->phase_pause) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_pp_" + std::to_string(ig->phase_pause) + "_infer.txt"); + break; + case 4: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_lp_" + std::to_string(ig->layer_pause) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_lp_" + std::to_string(ig->layer_pause) + "_infer.txt"); + break; + case 5: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_lp_" + std::to_string(ig->layer_pause) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_lp_" + std::to_string(ig->layer_pause) + "_infer.txt"); + break; + case 8: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_tp_" + std::to_string(ig->token_pause) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_tp_" + std::to_string(ig->token_pause) + "_infer.txt"); + break; + case 16: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_qi_" + std::to_string(ig->query_interval) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_qi_" + std::to_string(ig->query_interval) + "_infer.txt"); + break; + case 17: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_qi_" + std::to_string(ig->query_interval) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_qi_" + std::to_string(ig->query_interval) + "_infer.txt"); + break; + case 20: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_lp_" + std::to_string(ig->layer_pause) + "_qi_" + std::to_string(ig->query_interval) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_lp_" + std::to_string(ig->layer_pause) + "_qi_" + std::to_string(ig->query_interval) + "_infer.txt"); + break; + case 21: + output_hard = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_lp_" + std::to_string(ig->layer_pause) + "_qi_" + std::to_string(ig->query_interval) + "_hard.txt"); + output_infer = joinPaths(ig->output_dir, "stream_llama.cpp_" + std::to_string(ig->cpu_clk_idx_p) + "-" + std::to_string(ig->ram_clk_idx_p) + "_to_" + std::to_string(ig->cpu_clk_idx_d) + "-" + std::to_string(ig->ram_clk_idx_d) + "_lp_" + std::to_string(ig->layer_pause) + "_qi_" + std::to_string(ig->query_interval) + "_infer.txt"); + break; + case 3: + case 6: + case 7: + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + default: + std::cerr << "[ERROR] Not Controlled Configuration\n"; + return false; + } + + ig->fixed_config = fixed_config; + std::strcpy(ig->output_path_hard, output_hard.c_str()); + std::strcpy(ig->output_path_infer, output_infer.c_str()); + return true; +} + +void ignite_params_system_info(const llama_igparams * igparams) { + if (!igparams) { + return; + } + + printf("%s: device name\t\t\t= %s\n\r", __func__, igparams->device_name); + printf("%s: ignite active status\t\t= %s\n\r", __func__, igparams->is_ignite_active ? "ON" : "OFF"); + printf("%s: backend compute profile\t= %s\n\r", __func__, igparams->backend_compute_profile ? "ON" : "OFF"); + printf("%s: backend op breakdown\t\t= %s\n\r", __func__, igparams->backend_op_breakdown ? "ON" : "OFF"); + printf("%s: strict generation\t\t= %s\n\r", __func__, igparams->strict_limit ? "ON" : "OFF"); + printf("%s: enable thinking\t\t= %s\n\r", __func__, igparams->enable_thinking ? "ON" : "OFF"); + printf("%s: prefill CPU/RAM clock idx\t= %d / %d\n\r", __func__, igparams->cpu_clk_idx_p, igparams->ram_clk_idx_p); + printf("%s: decode CPU/RAM clock idx\t= %d / %d\n\r", __func__, igparams->cpu_clk_idx_d, igparams->ram_clk_idx_d); + printf("%s: input dataset path\t\t= %s\n\r", __func__, igparams->input_path); + printf("%s: resource output file\t\t= %s\n\r", __func__, igparams->output_path_hard); + printf("%s: llm output file\t\t= %s\n\r", __func__, igparams->output_path_infer); +} diff --git a/src/llama-ignite.h b/src/llama-ignite.h new file mode 100644 index 000000000..f88f17bee --- /dev/null +++ b/src/llama-ignite.h @@ -0,0 +1,32 @@ +#pragma once + +/* + * This file is written to manage ignite parameters in internal graph compute operations. + * Please refer to common_params in common.h for CLI-facing details. + */ + +#include +#include + +#include "llama.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_context; // opaque +struct llama_igparams; // opaque + +void llama_ignite_set_active(struct llama_context * ctx, bool active); +bool llama_ignite_get_active(struct llama_context * ctx); +void llama_ignite_set_layer_pause(struct llama_context * ctx, uint16_t ms); +uint16_t llama_ignite_get_layer_pause(struct llama_context * ctx); + +#ifdef __cplusplus +} +#endif + +bool init_ignite_params(struct llama_context * ctx, llama_igparams * igparams); +struct llama_igparams * get_ignite_params(struct llama_context * ctx); +bool init_ignite_filename(struct llama_context * ctx); +void ignite_params_system_info(const llama_igparams * igparams);