diff --git a/.github/scripts/build-test.sh b/.github/scripts/build-test.sh index 413c000b..729be318 100755 --- a/.github/scripts/build-test.sh +++ b/.github/scripts/build-test.sh @@ -12,13 +12,13 @@ echo "Starting build and test for platform $PLATFORM, compiler suite $COMPILER" # In vanilla containers we need to install the necessary development packages if [[ "$PLATFORM" == almalinux* ]]; then echo "Installing additional RPM development packages" - dnf install -y gcc-c++ cmake boost boost-devel make clang + dnf install -y gcc-c++ cmake boost boost-devel make clang libcurl-devel fi if [[ $PLATFORM == ubuntu* ]]; then echo "Installing additional DEB development packages" apt update - apt -y install g++ clang libboost-dev cmake make + apt -y install g++ clang libboost-dev cmake make libcurl4-openssl-dev fi # If environment variables CXX or CMAKE are defined then use these diff --git a/.github/scripts/flake8.sh b/.github/scripts/flake8.sh index b0ec4df3..4b9c8293 100755 --- a/.github/scripts/flake8.sh +++ b/.github/scripts/flake8.sh @@ -1,7 +1,9 @@ #! /bin/sh # First we need to install Python and Flake 8 -dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make +# N.B. this is for an Alma Linux container (specifically almalinux 9 +# but should be quite generic) +dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make libcurl-devel pip install flake8 # Configure and run flake8 target diff --git a/CMakeLists.txt b/CMakeLists.txt index 63118630..4eb3f82d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" # Both of these issues can be resolved using CMake's compile features, see # # - https://cmake.org/cmake/help/v3.3/manual/cmake-compile-features.7.html -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt index 0c8f24d6..2c9fa6cb 100644 --- a/package/CMakeLists.txt +++ b/package/CMakeLists.txt @@ -10,14 +10,19 @@ add_executable(prmon src/prmon.cpp src/countmon.cpp src/wallmon.cpp src/memmon.cpp + src/machmon.cpp + src/powermon.cpp ) target_link_libraries(prmon PRIVATE nlohmann_json::nlohmann_json spdlog::spdlog_header_only + CURL::libcurl dl ) +find_package(CURL REQUIRED) + if (BUILD_BENCHMARK_LOG) add_executable(benchmark-log benchmarks/benchmark-log.cpp) endif(BUILD_BENCHMARK_LOG) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index a3c253fc..fc9ccfe1 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -93,8 +93,9 @@ def make_net(proc_net, fixed_value, rand=False): print(random.randint(0, net_lim) if rand else fixed_value, file=f) -def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, - rand_sm_mem=False, rand_fb=False): +def make_nvidia( + proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem=False, rand_fb=False +): # idx print(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem, rand_fb) smi_fname = os.path.join(proc_nvidia, "smi") @@ -121,6 +122,76 @@ def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, print("", file=f) +def make_machstat(proc_mach, fixed_value, rand=False): + mach_fname = os.path.join(proc_mach, "stat") + lim = 2**64 - 1 + if rand: + user, nice, system = random.randint(1, lim) + else: + user, nice, system = fixed_value, fixed_value, fixed_value + with open(mach_fname, "w") as f: + print("cpu", file=f, end=" ") # Relevant line + # 1 utime - Time in user mode + # 2 ntime - Time in nice mode + # 3 stime - Time in system mode + print(f"{user} {nice} {system}", file=f, end=" ") + + # 4 - 10 are never used, set to 0 + print("0 " * 7, file=f, end="") + + +def make_power(power_dir, fixed_value, rand=False): + power_fname = os.path.join(power_dir, "ipmi.prom") + if rand: + dcmi, ipmi, hwmon = (random.randint(0, 1000) for _ in range(3)) + rapl_0, rapl_1 = (random.randint(0, 1000000) for _ in range(3)) + else: + dcmi = ipmi = hwmon = rapl_0 = rapl_1 = fixed_value + + with open(power_fname, "w") as f: + print( + "# HELP node_hwmon_power_watt " "Hardware monitor for power consumption", + file=f, + ) + print("# TYPE node_hwmon_power_watt gauge", file=f) + print( + "node_hwmon_power_watt" + f'{{chip="power_supply_bat1",sensor="power1"}} {hwmon}', + file=f, + ) + + print( + "# HELP node_rapl_package_joules_total " + "Current RAPL package value in joules", + file=f, + ) + print("# TYPE node_rapl_package_joules_total counter", file=f) + print( + "node_rapl_package_joules_total" + '{index="0",path="/sys/class/powercap/intel-rapl:0"}' + f" {rapl_0}", + file=f, + ) + print( + "node_rapl_package_joules_total" + '{index="1",path="/sys/class/powercap/intel-rapl:1"}' + f" {rapl_1}", + file=f, + ) + + print( + "# HELP ipmi_dcmi_power_consumption_watts " + "Current power consumption in watts", + file=f, + ) + print("# TYPE ipmi_dcmi_power_consumption_watts gauge", file=f) + print(f"ipmi_dcmi_power_consumption_watts {dcmi}", file=f) + + print("# HELP ipmi_power_watts " "Power reading of the sensor in watts", file=f) + print("# TYPE ipmi_power_watts gauge", file=f) + print('ipmi_power_watts{id="1",name="Pwr Consumption"}' f" {ipmi}", file=f) + + def createPath(new_dir): dir_name = os.path.join(os.getcwd(), "precooked_tests") if not os.path.isdir(dir_name): @@ -151,27 +222,35 @@ def createTestMonotonic(): proc_pid = os.path.join(proc, str(pid)) net_dir = os.path.join(new_dir, "net") nvidia_dir = os.path.join(new_dir, "nvidia") + power_dir = os.path.join(new_dir, "power") os.mkdir(proc_pid) os.mkdir(net_dir) os.mkdir(nvidia_dir) + os.mkdir(power_dir) if i == 1: make_stat(pid, proc_pid, 5000000) make_io(proc_pid, 500) make_smaps(proc_pid, 5000) make_net(net_dir, 500000) make_nvidia(nvidia_dir, 50, 50) + make_machstat(proc, 50000) + make_power(power_dir, 500) elif i == 2: make_stat(pid, proc_pid, 10000000) make_io(proc_pid, 1000) make_smaps(proc_pid, 10000) make_net(net_dir, 1000000) make_nvidia(nvidia_dir, 100, 100) + make_machstat(proc, 100000) + make_power(power_dir, 1000) else: make_stat(pid, proc_pid, 2000000) make_io(proc_pid, 200) make_smaps(proc_pid, 2000) make_net(net_dir, 200000) make_nvidia(nvidia_dir, "-", 20) + make_machstat(proc, 20000) + make_power(power_dir, 200) gc.collect() time.sleep(interval) i += 1 @@ -195,14 +274,19 @@ def createTestRand(dir_name, iter): proc_pid = os.path.join(proc, str(pid)) net_dir = os.path.join(new_dir, "net") nvidia_dir = os.path.join(new_dir, "nvidia") + power_dir = os.path.join(new_dir, "power") os.mkdir(proc_pid) os.mkdir(net_dir) os.mkdir(nvidia_dir) + os.mkdir(proc) + os.mkdir(power_dir) make_stat(pid, proc_pid, 0, True) make_io(proc_pid, 0, True) make_smaps(proc_pid, 0, True) make_net(net_dir, 0, True) make_nvidia(nvidia_dir, 0, True) + make_machstat(proc, 0, True) + make_power(power_dir, 0, True) gc.collect() time.sleep(interval) diff --git a/package/scripts/precooked_tests/drop/1/power/ipmi.prom b/package/scripts/precooked_tests/drop/1/power/ipmi.prom new file mode 100644 index 00000000..3e2290da --- /dev/null +++ b/package/scripts/precooked_tests/drop/1/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 500 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 500 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 500 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 500 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 500 diff --git a/package/scripts/precooked_tests/drop/1/proc/stat b/package/scripts/precooked_tests/drop/1/proc/stat new file mode 100644 index 00000000..7d47cd91 --- /dev/null +++ b/package/scripts/precooked_tests/drop/1/proc/stat @@ -0,0 +1 @@ +cpu 50000 50000 50000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/scripts/precooked_tests/drop/2/power/ipmi.prom b/package/scripts/precooked_tests/drop/2/power/ipmi.prom new file mode 100644 index 00000000..264e28a0 --- /dev/null +++ b/package/scripts/precooked_tests/drop/2/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 1000 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 1000 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 1000 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 1000 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 1000 diff --git a/package/scripts/precooked_tests/drop/2/proc/stat b/package/scripts/precooked_tests/drop/2/proc/stat new file mode 100644 index 00000000..dd6a9913 --- /dev/null +++ b/package/scripts/precooked_tests/drop/2/proc/stat @@ -0,0 +1 @@ +cpu 100000 100000 100000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/scripts/precooked_tests/drop/3/nvidia/smi b/package/scripts/precooked_tests/drop/3/nvidia/smi index 07cbf9c8..c17b14f4 100644 --- a/package/scripts/precooked_tests/drop/3/nvidia/smi +++ b/package/scripts/precooked_tests/drop/3/nvidia/smi @@ -1 +1 @@ -0 1729 G - - - - - - 20 0 python3 +0 1729 G - - - - - - 20 0 python3 diff --git a/package/scripts/precooked_tests/drop/3/power/ipmi.prom b/package/scripts/precooked_tests/drop/3/power/ipmi.prom new file mode 100644 index 00000000..94665716 --- /dev/null +++ b/package/scripts/precooked_tests/drop/3/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 200 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 200 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 200 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 200 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 200 diff --git a/package/scripts/precooked_tests/drop/3/proc/stat b/package/scripts/precooked_tests/drop/3/proc/stat new file mode 100644 index 00000000..6b0b2b08 --- /dev/null +++ b/package/scripts/precooked_tests/drop/3/proc/stat @@ -0,0 +1 @@ +cpu 20000 20000 20000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/src/machmon.cpp b/package/src/machmon.cpp new file mode 100644 index 00000000..39346ac9 --- /dev/null +++ b/package/src/machmon.cpp @@ -0,0 +1,106 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +#include "machmon.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +#define MONITOR_NAME "machmon" + +// Constructor; uses RAII pattern to be valid after construction +machmon::machmon() : mach_stats{} { + log_init(MONITOR_NAME); +#undef MONITOR_NAME + for (const auto& param : params) { + mach_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + } + + valid = true; +} + +// Read the first line of /proc/stat and return {utime, stime, ntime} + +static std::tuple read_proc_stat(const std::string read_path = "") { + const std::string stat_path = read_path + "/proc/stat"; + std::ifstream proc_stat{stat_path}; + if (!proc_stat) { + throw std::runtime_error("Cannot open " + stat_path); + } + + std::string cpu_label{}; + long user{}, nice{}, system{}; + + // First line: "cpu user nice system idle ..." + if (!(proc_stat >> cpu_label >> user >> nice >> system)) { + throw std::runtime_error("Unexpected format in " + stat_path); + } + + return std::make_tuple(user, system, nice); +} + +void machmon::update_stats(const std::vector& /*pids*/, + const std::string read_path) { + if (!valid) { + return; + } + + long utime, stime, ntime; + std::tie(utime, stime, ntime) = read_proc_stat(read_path); + + // Subtract initial snapshot on first read + if (!initialised) { + initial_utime = utime; + initial_stime = stime; + initial_ntime = ntime; + initialised = true; + } + + const long clk = sysconf(_SC_CLK_TCK); + mach_stats.at("utime_total").set_value(std::max(0L, utime - initial_utime) / clk); + mach_stats.at("stime_total").set_value(std::max(0L, stime - initial_stime) / clk); + mach_stats.at("ntime_total").set_value(std::max(0L, ntime - initial_ntime) / clk); +} + +// Return the summed counters +prmon::monitored_value_map const machmon::get_text_stats() { + prmon::monitored_value_map cpu_stat_map{}; + for (const auto& value : mach_stats) { + cpu_stat_map[value.first] = value.second.get_value(); + } + return cpu_stat_map; +} + +// Same for JSON +prmon::monitored_value_map const machmon::get_json_total_stats() { + return machmon::get_text_stats(); +} + +// No averages for global CPU time either +prmon::monitored_average_map const machmon::get_json_average_stats( + unsigned long long /*elapsed_clock_ticks*/) { + static const prmon::monitored_average_map empty_average_stats{}; + return empty_average_stats; +} + +// Return the parameter list +prmon::parameter_list const machmon::get_parameter_list() { return params; } + +// No extra hardware information needed here +void const machmon::get_hardware_info(nlohmann::json& /*hw_json*/) { + return; +} + +void const machmon::get_unit_info(nlohmann::json& unit_json) { + prmon::fill_units(unit_json, params); + return; +} \ No newline at end of file diff --git a/package/src/machmon.h b/package/src/machmon.h new file mode 100644 index 00000000..662cc0d0 --- /dev/null +++ b/package/src/machmon.h @@ -0,0 +1,58 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +// Full CPU monitoring class + +#ifndef PRMON_MACHMON_H +#define PRMON_MACHMON_H 1 + +#include +#include +#include +#include + +#include "Imonitor.h" +#include "MessageBase.h" +#include "parameter.h" +#include "registry.h" + +class machmon final : public Imonitor, public MessageBase { + private: + // Setup the parameters to monitor here + const prmon::parameter_list params = { + {"utime_total", "s", ""}, + {"stime_total", "s", ""}, + {"ntime_total", "s", ""}, + }; + + // Dynamic monitoring container + prmon::monitored_list mach_stats; + + long initial_utime{0L}; + long initial_stime{0L}; + long initial_ntime{0L}; + bool initialised{false}; + bool valid{false}; // To disable monitor if /proc/stat is not readable + + public: + machmon(); + + void update_stats(const std::vector& pids, + const std::string read_path = ""); + + // These are the stat getter methods which retrieve current statistics + prmon::monitored_value_map const get_text_stats(); + prmon::monitored_value_map const get_json_total_stats(); + prmon::monitored_average_map const get_json_average_stats( + unsigned long long elapsed_clock_ticks); + prmon::parameter_list const get_parameter_list(); + + // Hardware information getter to run once + void const get_hardware_info(nlohmann::json& hw_json); + void const get_unit_info(nlohmann::json& unit_json); + bool const is_valid() { return valid; } +}; +REGISTER_MONITOR(Imonitor, machmon, "Monitor full machine cpu time") + + +#endif // PRMON_MACHMON_H \ No newline at end of file diff --git a/package/src/powermon.cpp b/package/src/powermon.cpp new file mode 100644 index 00000000..1e8ae100 --- /dev/null +++ b/package/src/powermon.cpp @@ -0,0 +1,406 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +#include "powermon.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +#include "utils.h" + +#define MONITOR_NAME "powermon" + +// Maximum response size (8 MB) +static const curl_off_t kMaxResponseSize = 8 * 1024 * 1024; + +// libcurl write, appending data to a std::string +static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, + void* userdata) { + auto* buf = reinterpret_cast(userdata); + const size_t new_size = size * nmemb; + + // Return 0 if the new size would exceed the maximum allowed response size + if (buf->size() + new_size > static_cast(kMaxResponseSize)) return 0; + + buf->append(ptr, new_size); + return new_size; +} + +// Constructor; uses RAII pattern to be valid +// after construction +powermon::powermon() : power_stats{} { + log_init(MONITOR_NAME); +#undef MONITOR_NAME + + valid = false; // Default value, only set to true if everything works + + // Global init for curl, explicitly + if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) { + error("Failed to initialise libcurl"); + return; + } + // Initialize parameters + for (const auto& param : params) { + // RAPL values are monotonic, so we set the flag to true. Else, false. + if (param.get_name() == "rapl_1" || param.get_name() == "rapl_2") { + power_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + } else { + power_stats.emplace(param.get_name(), prmon::monitored_value(param, false)); + } + } + + // Function to check urls/directories and store them + auto add_entry = [&](const std::string& entry){ + if (entry.empty()) return; + if (entry.find("://") != std::string::npos) + metrics_urls.push_back(entry); + else + metrics_dirs.push_back(entry); + }; + + // PROMETHEUS_LOCAL_URL: comma separated list of exporter URLs or directories + // Expected format: "http://localhost:9100/metrics,/node_exporter/ipmi.prom" + const char* env_string = std::getenv("PROMETHEUS_LOCAL_URL"); + + // Read through env_string, push each to either url or directory + if (env_string) { + unsigned pos{0}, start{0}; + while (true) { + if (env_string[pos] == ',' || env_string[pos] == '\0') { + add_entry(std::string(env_string + start, pos - start)); + if (env_string[pos] == '\0') break; + start = ++pos; + } else { + ++pos; + } + } + } + + // Use default urls if no url was provided in the environment variable + if (metrics_urls.empty()) { + metrics_urls = default_metrics_urls; + debug("No exporter URLs provided, using default powermon ports."); + }; + + // Use default dirs if no dir was provided in the environment variable + if (metrics_dirs.empty()) { + metrics_dirs = default_metrics_dirs; + debug("No power directories provided, using default powermon dirs."); + }; + + // Check at least one endpoint answers + const std::vector bodies = gather_bodies(""); + if (bodies.empty()){ + info("Cannot read power metrics from any exporter or directory." + "powermon disabled."); + return; + } + + // Check at least a metric is available + std::vector dcmi, rapl, ipmi, hwmon; + for (const auto& body : bodies) collect_lines(body, dcmi, rapl, ipmi, hwmon); + if (dcmi.empty() && rapl.empty() && ipmi.empty() && hwmon.empty()) { + info("Exporter reachable but none of the dcmi/rapl/ipmi/hwmon metrics " + "were found. powermon disabled."); + return; + } + + // If none of the errors are caught, then valid is set to true. + valid = true; +} + +// Destructor: closes libcurl global state +powermon::~powermon() { curl_global_cleanup(); } + +// Fetch Prometheus text from the ctt URL. Returns "" on any failure. +std::string powermon::fetch_metrics(const std::string& url) { + std::string response{}; + CURL* curl = curl_easy_init(); + if (!curl) { + error("Failed to initialise libcurl, curl_easy_init() returned NULL"); + return {}; + } + + // Define a function to check if any response is != 0 + CURLcode rc = CURLE_OK; + auto set_ok = [&](CURLoption option, auto value) { + rc = curl_easy_setopt(curl, option, value); + if (rc != CURLE_OK) { + error("curl_easy_setopt failed: " + std::string(curl_easy_strerror(rc))); + curl_easy_cleanup(curl); + valid = false; + return false; + }; + return true; + }; + + // Configure curl options. If any configuration fails, clean and return. + if (!set_ok(CURLOPT_URL, url.c_str()) || + !set_ok(CURLOPT_WRITEFUNCTION, curl_write_cb) || + !set_ok(CURLOPT_WRITEDATA, &response) || + !set_ok(CURLOPT_MAXFILESIZE_LARGE, kMaxResponseSize) || +#if LIBCURL_VERSION_NUM >= 0x075500 // 7.85.0 + !set_ok(CURLOPT_PROTOCOLS_STR, "http") || +#else + !set_ok(CURLOPT_PROTOCOLS, static_cast(CURLPROTO_HTTP)) || +#endif + !set_ok(CURLOPT_CONNECTTIMEOUT, 1L) || + !set_ok(CURLOPT_TIMEOUT, 2L) || + !set_ok(CURLOPT_PROXY, "") || + !set_ok(CURLOPT_NOSIGNAL, 1L)) { + curl_easy_cleanup(curl); + return {}; + }; + + // Execute curl, check filesize + CURLcode curl_response = curl_easy_perform(curl); + if (curl_response == CURLE_FILESIZE_EXCEEDED || curl_response == CURLE_WRITE_ERROR) { + error("Response from " + url + "exceeds the " + + std::to_string(kMaxResponseSize) + "byte limit, discarding it"); + return {}; + } + return response; +}; + +// Read Prometheus text from a text file. +std::string powermon::read_metrics_file(const std::string& path){ + std::ifstream f{path}; + if (!f) return {}; // Missing file + + f.seekg(0, std::ios::end); + const std::streamoff size = f.tellg(); + if (size < 0 || size > kMaxResponseSize) { + error("File " + path + " exceeds the " + std::to_string(kMaxResponseSize) + + " byte limit, ignoring it"); + return {}; + } + f.seekg(0); + + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// Collect Prometheus-style text from, in order of priority: +// 1. exporter URLs (PROMETHEUS_LOCAL_URL or default) +// 2. directories from PROMETHEUS_LOCAL_URL +// 3. default directory /power +// First source that works is used. +std::vector powermon::gather_bodies(const std::string& read_path){ + std::vector bodies; + + // 1. URLs (skipped if locked to directories) + if (source != Source::Files) { + for (const auto& url : metrics_urls) { + const std::string body = fetch_metrics(url); + if (!body.empty()) bodies.push_back(body); + } + if(!bodies.empty()) { + if (source == Source::Undecided){ + info("powermon: reading metrics from exporter URLs"); + source = Source::Http; + return bodies; + } + } + // Locked to HTTP and failed. + if (source == Source::Http) return bodies; + debug("powermon: reading from URLs failed"); + } + + // 2. Directories (if URLs failed or directory chosen) + std::vector dirs = metrics_dirs; + for (const auto& dir : dirs) { + debug("powermon: testing directory " + dir); + const std::string body = read_metrics_file(dir); + if (!body.empty()) bodies.push_back(body); + if (!bodies.empty()) { + if (source == Source::Undecided){ + info("powermon: reading metrics from directory " + dir); + source = Source::Files; + break; + } + } else { + debug("powermon: directory " + dir + " did not contain any metric or could not be read."); + } + + } + return bodies; +} + +// Parse a single Prometheus line: "metric_name{labels} value timestamp" +// Returns the value if the line starts with metric_name, otherwise -1.0 +double powermon::parse_metric_line(const std::string& line, + const std::string& metric_name) const { + if (line.empty() || line[0] == '#') return -1.0; // Skip header or empty lines + if (line.rfind(metric_name, 0) != 0) return -1.0; // Skip lines without metric_name + + char next = line[metric_name.size()]; + if (next != '{' && next != ' ' && next != '\t') return -1.0; + + size_t value_start = line.find('}'); + if (value_start == std::string::npos) + value_start = metric_name.size(); + else + value_start += 1; + + while (value_start < line.size() && std::isspace(static_cast(line[value_start]))) + ++value_start; + + try { + const double val = std::stod(line.substr(value_start)); + // Return -1 if val is NaN, Inf, negative or over the maximum + if (std::isnan(val) || std::isinf(val)) return -1.0; + if (val < 0.0) return -1.0; + if (val >= static_cast(std::numeric_limits::max())) return -1.0; + return val; + } catch (...) { + return -1.0; + } +} + +// Extract the rellevant lines from the text body +void powermon::collect_lines(const std::string& body, + std::vector& dcmi, + std::vector& rapl, + std::vector& ipmi, + std::vector& hwmon) const { + std::istringstream stream{body}; + std::string line{}; + while (std::getline(stream, line)) { + std::string lower = line; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (lower.find("ipmi_dcmi_power_consumption_watts") != std::string::npos) + dcmi.push_back(line); + if (lower.find("node_rapl_package_joules_total") != std::string::npos) + rapl.push_back(line); + if (lower.find("ipmi_power_watts") != std::string::npos || + lower.find("ipmi_power_current_watts") != std::string::npos) + ipmi.push_back(line); + if (lower.find("node_hwmon_power_watt") != std::string::npos || + lower.find("node_hwmon_power_average_watt") != std::string::npos) + hwmon.push_back(line); + } +} + + +void powermon::update_stats(const std::vector& /*pids*/, + const std::string read_path) { + // Attempt to read from each metrics_url + const std::vector bodies = gather_bodies(read_path); + if (bodies.empty()) { + info("No power metrics from any exporter or directory, disabling powermon"); + valid = false; + return; + } + + // Collect matching lines from all sources + std::vector dcmi_lines{}, rapl_lines{}, ipmi_lines{}, hwmon_lines{}; + for (const auto& body : bodies) { + collect_lines(body, dcmi_lines, rapl_lines, ipmi_lines, hwmon_lines); + } + + // DCMI: sum all matching lines + double dcmi = 0.0; + try { + for (const auto& line : dcmi_lines) { + double val = parse_metric_line(line, "ipmi_dcmi_power_consumption_watts"); + if (val >= 0.0) dcmi += val; + } + } catch (...) { dcmi = 0.0; } + + // RAPL: penultimate line -> rapl_1, last line -> rapl_2 + double rapl_1 = 0.0, rapl_2 = 0.0; + try { + if (rapl_lines.size() >= 2) { + double v1 = parse_metric_line(rapl_lines[rapl_lines.size() - 2], + "node_rapl_package_joules_total"); + double v2 = parse_metric_line(rapl_lines[rapl_lines.size() - 1], + "node_rapl_package_joules_total"); + if (v1 >= 0.0) rapl_1 = v1; + if (v2 >= 0.0) rapl_2 = v2; + } else if (rapl_lines.size() == 1) { + double v = parse_metric_line(rapl_lines[0], "node_rapl_package_joules_total"); + if (v >= 0.0) rapl_1 = v; + } + } catch (...) { rapl_1 = rapl_2 = 0.0; } + + // IPMI: last matching line + double ipmi = 0.0; + try { + if (!ipmi_lines.empty()) { + double val = parse_metric_line(ipmi_lines.back(), "ipmi_power_watts"); + if (val < 0.0) val = parse_metric_line(ipmi_lines.back(), "ipmi_power_current_watts"); + if (val >= 0.0) ipmi = val; + } + } catch (...) { ipmi = 0.0; } + + // HWMON: last matching line + double hwmon = 0.0; + try { + if (!hwmon_lines.empty()) { + double val = parse_metric_line(hwmon_lines.back(), "node_hwmon_power_average_watt"); + if (val >= 0.0) hwmon = val; + } + } catch (...) { hwmon = 0.0; } + + // store function to handle value range checking and storing in power_stats + auto store = [&](const char* key, double value) { + if (!(value >= 0.0 && value < static_cast(std::numeric_limits::max()))) { + error(std::string("Value for ") + key + " is out of range, ignoring"); + value = 0.0; + } + power_stats.at(key).set_value(static_cast(value)); + }; + + try { + store("dcmi", dcmi); + store("rapl_1", rapl_1); + store("rapl_2", rapl_2); + store("ipmi", ipmi); + store("hwmon", hwmon); + } catch (const std::exception& e) { + error(std::string("Error storing metrics: ") + e.what()); + } +} + + +// Return the monitored values +prmon::monitored_value_map const powermon::get_text_stats() { + prmon::monitored_value_map stat_map{}; + for (const auto& value : power_stats) { + stat_map[value.first] = value.second.get_value(); + } + return stat_map; +} + +prmon::monitored_value_map const powermon::get_json_total_stats() { + return powermon::get_text_stats(); +} + +prmon::monitored_average_map const powermon::get_json_average_stats( + unsigned long long /*elapsed_clock_ticks*/) { + static const prmon::monitored_average_map empty{}; + return empty; +} + +prmon::parameter_list const powermon::get_parameter_list() { return params; } + +void const powermon::get_hardware_info(nlohmann::json& /*hw_json*/) { return; } + +void const powermon::get_unit_info(nlohmann::json& unit_json) { + prmon::fill_units(unit_json, params); + return; +} \ No newline at end of file diff --git a/package/src/powermon.h b/package/src/powermon.h new file mode 100644 index 00000000..f4f94127 --- /dev/null +++ b/package/src/powermon.h @@ -0,0 +1,91 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +// Power monitoring class + +#ifndef PRMON_POWERMON_H +#define PRMON_POWERMON_H 1 + +#include +#include +#include +#include + +#include "Imonitor.h" +#include "MessageBase.h" +#include "parameter.h" +#include "registry.h" +#include + +class powermon final : public Imonitor, public MessageBase { + private: + const prmon::parameter_list params = { + {"dcmi", "W", "DCMI power consumption"}, + {"ipmi", "W", "IPMI power consumption"}, + {"hwmon", "W", "HWMON power consumption"}, + {"rapl_1", "J", "RAPL package 1 energy in joules"}, + {"rapl_2", "J", "RAPL package 2 energy in joules"} + }; + + // Default urls to look at + const std::vector default_metrics_urls{ + "http://localhost:9100/metrics", + "http://localhost:9290/metrics" + }; + std::vector metrics_urls; + + // Directories to look at + const std::vector default_metrics_dirs{ + "/power/ipmi.prom" + }; + std::vector metrics_dirs; + + // Source of metrics, decided first time data is found + enum class Source {Undecided, Http, Files}; + Source source{Source::Undecided}; + +// // Check and choose data source +// void source_check(); + + // Extract lines from the Prometheus text output for each metric type + void collect_lines(const std::string& body, + std::vector& dcmi, + std::vector& rapl, + std::vector& ipmi, + std::vector& hwmon) const; + + bool valid{false}; // To disable monitor if all ports fail or no metric is read + + prmon::monitored_list power_stats; + + std::string fetch_metrics(const std::string& url); + double parse_metric_line(const std::string& line, + const std::string& metric_name) const; + + public: + powermon(); + ~powermon(); + + void update_stats(const std::vector& pids, + const std::string read_path = ""); + + // These are the stat getter methods which retrieve current statistics + prmon::monitored_value_map const get_text_stats(); + prmon::monitored_value_map const get_json_total_stats(); + prmon::monitored_average_map const get_json_average_stats( + unsigned long long elapsed_clock_ticks); + prmon::parameter_list const get_parameter_list(); + + // Additional function to read from a directory instead of curl + std::string read_metrics_file(const std::string& read_path = ""); + std::vector gather_bodies(const std::string& read_path); + + // This is the hardware information getter that runs once + void const get_hardware_info(nlohmann::json& hw_json); + void const get_unit_info(nlohmann::json& unit_json); + bool const is_valid() { return valid; } +}; +REGISTER_MONITOR(Imonitor, powermon, "Monitor power consumption - prometheus") + + +#endif // PRMON_POWERMON_H