Power consumption monitoring

.github/scripts/build-test.sh

@@ -12,13 +12,13 @@ echo "Starting build and test for platform $PLATFORM, compiler suite $COMPILER"
 # In vanilla containers we need to install the necessary development packages
 if [[ "$PLATFORM" == almalinux* ]]; then
     echo "Installing additional RPM development packages"
-    dnf install -y gcc-c++ cmake boost boost-devel make clang
+    dnf install -y gcc-c++ cmake boost boost-devel make clang libcurl-devel
 fi
 
 if [[ $PLATFORM == ubuntu* ]]; then
     echo "Installing additional DEB development packages"
     apt update
-    apt -y install g++ clang libboost-dev cmake make
+    apt -y install g++ clang libboost-dev cmake make libcurl4-openssl-dev
 fi
 
 # If environment variables CXX or CMAKE are defined then use these

.github/scripts/flake8.sh

@@ -1,7 +1,9 @@
 #! /bin/sh
 
 # First we need to install Python and Flake 8
-dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make
+# N.B. this is for an Alma Linux container (specifically almalinux 9
+# but should be quite generic)
+dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make libcurl-devel
 pip install flake8
 
 # Configure and run flake8 target

CMakeLists.txt

@@ -78,7 +78,7 @@ set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}"
 # Both of these issues can be resolved using CMake's compile features, see
 #
 # - https://cmake.org/cmake/help/v3.3/manual/cmake-compile-features.7.html
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 

package/CMakeLists.txt

@@ -10,14 +10,19 @@ add_executable(prmon src/prmon.cpp
            src/countmon.cpp
            src/wallmon.cpp
            src/memmon.cpp
+           src/machmon.cpp
+           src/powermon.cpp
            )
 
 target_link_libraries(prmon PRIVATE
            nlohmann_json::nlohmann_json
            spdlog::spdlog_header_only
+           CURL::libcurl
            dl
            )
 
+find_package(CURL REQUIRED)
+
 if (BUILD_BENCHMARK_LOG)
   add_executable(benchmark-log benchmarks/benchmark-log.cpp)
 endif(BUILD_BENCHMARK_LOG)

package/scripts/precook_test.py

@@ -93,8 +93,9 @@ def make_net(proc_net, fixed_value, rand=False):
             print(random.randint(0, net_lim) if rand else fixed_value, file=f)
 
 
-def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb,
-                rand_sm_mem=False, rand_fb=False):
+def make_nvidia(
+    proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem=False, rand_fb=False
+):
     # idx
     print(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem, rand_fb)
     smi_fname = os.path.join(proc_nvidia, "smi")
@@ -121,6 +122,76 @@ def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb,
         print("", file=f)
 
 
+def make_machstat(proc_mach, fixed_value, rand=False):
+    mach_fname = os.path.join(proc_mach, "stat")
+    lim = 2**64 - 1
+    if rand:
+        user, nice, system = random.randint(1, lim)
+    else:
+        user, nice, system = fixed_value, fixed_value, fixed_value
+    with open(mach_fname, "w") as f:
+        print("cpu", file=f, end=" ")  # Relevant line
+        # 1 utime - Time in user mode
+        # 2 ntime - Time in nice mode
+        # 3 stime - Time in system mode
+        print(f"{user} {nice} {system}", file=f, end=" ")
+
+        # 4 - 10 are never used, set to 0
+        print("0 " * 7, file=f, end="")
+
+
+def make_power(power_dir, fixed_value, rand=False):
+    power_fname = os.path.join(power_dir, "ipmi.prom")
+    if rand:
+        dcmi, ipmi, hwmon = (random.randint(0, 1000) for _ in range(3))
+        rapl_0, rapl_1 = (random.randint(0, 1000000) for _ in range(3))
+    else:
+        dcmi = ipmi = hwmon = rapl_0 = rapl_1 = fixed_value
+
+    with open(power_fname, "w") as f:
+        print(
+            "# HELP node_hwmon_power_watt " "Hardware monitor for power consumption",
+            file=f,
+        )
+        print("# TYPE node_hwmon_power_watt gauge", file=f)
+        print(
+            "node_hwmon_power_watt"
+            f'{{chip="power_supply_bat1",sensor="power1"}} {hwmon}',
+            file=f,
+        )
+
+        print(
+            "# HELP node_rapl_package_joules_total "
+            "Current RAPL package value in joules",
+            file=f,
+        )
+        print("# TYPE node_rapl_package_joules_total counter", file=f)
+        print(
+            "node_rapl_package_joules_total"
+            '{index="0",path="/sys/class/powercap/intel-rapl:0"}'
+            f" {rapl_0}",
+            file=f,
+        )
+        print(
+            "node_rapl_package_joules_total"
+            '{index="1",path="/sys/class/powercap/intel-rapl:1"}'
+            f" {rapl_1}",
+            file=f,
+        )
+
+        print(
+            "# HELP ipmi_dcmi_power_consumption_watts "
+            "Current power consumption in watts",
+            file=f,
+        )
+        print("# TYPE ipmi_dcmi_power_consumption_watts gauge", file=f)
+        print(f"ipmi_dcmi_power_consumption_watts {dcmi}", file=f)
+
+        print("# HELP ipmi_power_watts " "Power reading of the sensor in watts", file=f)
+        print("# TYPE ipmi_power_watts gauge", file=f)
+        print('ipmi_power_watts{id="1",name="Pwr Consumption"}' f" {ipmi}", file=f)
+
+
 def createPath(new_dir):
     dir_name = os.path.join(os.getcwd(), "precooked_tests")
     if not os.path.isdir(dir_name):
@@ -151,27 +222,35 @@ def createTestMonotonic():
         proc_pid = os.path.join(proc, str(pid))
         net_dir = os.path.join(new_dir, "net")
         nvidia_dir = os.path.join(new_dir, "nvidia")
+        power_dir = os.path.join(new_dir, "power")
         os.mkdir(proc_pid)
         os.mkdir(net_dir)
         os.mkdir(nvidia_dir)
+        os.mkdir(power_dir)
         if i == 1:
             make_stat(pid, proc_pid, 5000000)
             make_io(proc_pid, 500)
             make_smaps(proc_pid, 5000)
             make_net(net_dir, 500000)
             make_nvidia(nvidia_dir, 50, 50)
+            make_machstat(proc, 50000)
+            make_power(power_dir, 500)
         elif i == 2:
             make_stat(pid, proc_pid, 10000000)
             make_io(proc_pid, 1000)
             make_smaps(proc_pid, 10000)
             make_net(net_dir, 1000000)
             make_nvidia(nvidia_dir, 100, 100)
+            make_machstat(proc, 100000)
+            make_power(power_dir, 1000)
         else:
             make_stat(pid, proc_pid, 2000000)
             make_io(proc_pid, 200)
             make_smaps(proc_pid, 2000)
             make_net(net_dir, 200000)
             make_nvidia(nvidia_dir, "-", 20)
+            make_machstat(proc, 20000)
+            make_power(power_dir, 200)
         gc.collect()
         time.sleep(interval)
         i += 1
@@ -195,14 +274,19 @@ def createTestRand(dir_name, iter):
         proc_pid = os.path.join(proc, str(pid))
         net_dir = os.path.join(new_dir, "net")
         nvidia_dir = os.path.join(new_dir, "nvidia")
+        power_dir = os.path.join(new_dir, "power")
         os.mkdir(proc_pid)
         os.mkdir(net_dir)
         os.mkdir(nvidia_dir)
+        os.mkdir(proc)
+        os.mkdir(power_dir)
         make_stat(pid, proc_pid, 0, True)
         make_io(proc_pid, 0, True)
         make_smaps(proc_pid, 0, True)
         make_net(net_dir, 0, True)
         make_nvidia(nvidia_dir, 0, True)
+        make_machstat(proc, 0, True)
+        make_power(power_dir, 0, True)
         gc.collect()
         time.sleep(interval)
 

package/scripts/precooked_tests/drop/1/power/ipmi.prom

@@ -0,0 +1,13 @@
+# HELP node_hwmon_power_watt Hardware monitor for power consumption
+# TYPE node_hwmon_power_watt gauge
+node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 500
+# HELP node_rapl_package_joules_total Current RAPL package value in joules
+# TYPE node_rapl_package_joules_total counter
+node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 500
+node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 500
+# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts
+# TYPE ipmi_dcmi_power_consumption_watts gauge
+ipmi_dcmi_power_consumption_watts 500
+# HELP ipmi_power_watts Power reading of the sensor in watts
+# TYPE ipmi_power_watts gauge
+ipmi_power_watts{id="1",name="Pwr Consumption"} 500

package/scripts/precooked_tests/drop/1/proc/stat

@@ -0,0 +1 @@
+cpu 50000 50000 50000 0 0 0 0 0 0 0 

package/scripts/precooked_tests/drop/2/power/ipmi.prom

@@ -0,0 +1,13 @@
+# HELP node_hwmon_power_watt Hardware monitor for power consumption
+# TYPE node_hwmon_power_watt gauge
+node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 1000
+# HELP node_rapl_package_joules_total Current RAPL package value in joules
+# TYPE node_rapl_package_joules_total counter
+node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 1000
+node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 1000
+# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts
+# TYPE ipmi_dcmi_power_consumption_watts gauge
+ipmi_dcmi_power_consumption_watts 1000
+# HELP ipmi_power_watts Power reading of the sensor in watts
+# TYPE ipmi_power_watts gauge
+ipmi_power_watts{id="1",name="Pwr Consumption"} 1000

package/scripts/precooked_tests/drop/2/proc/stat

@@ -0,0 +1 @@
+cpu 100000 100000 100000 0 0 0 0 0 0 0 

package/scripts/precooked_tests/drop/3/nvidia/smi

@@ -1 +1 @@
-0 1729 G - - - - - - 20 0 python3
+0 1729 G - - - - - - 20 0 python3 

package/scripts/precooked_tests/drop/3/power/ipmi.prom

@@ -0,0 +1,13 @@
+# HELP node_hwmon_power_watt Hardware monitor for power consumption
+# TYPE node_hwmon_power_watt gauge
+node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 200
+# HELP node_rapl_package_joules_total Current RAPL package value in joules
+# TYPE node_rapl_package_joules_total counter
+node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 200
+node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 200
+# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts
+# TYPE ipmi_dcmi_power_consumption_watts gauge
+ipmi_dcmi_power_consumption_watts 200
+# HELP ipmi_power_watts Power reading of the sensor in watts
+# TYPE ipmi_power_watts gauge
+ipmi_power_watts{id="1",name="Pwr Consumption"} 200

package/scripts/precooked_tests/drop/3/proc/stat

@@ -0,0 +1 @@
+cpu 20000 20000 20000 0 0 0 0 0 0 0 

package/src/machmon.cpp

@@ -0,0 +1,106 @@
+// Copyright (C) 2018-2025 CERN
+// License Apache2 - see LICENCE file
+
+#include "machmon.h"
+
+#include <unistd.h>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <tuple>
+
+#include "utils.h"
+
+#define MONITOR_NAME "machmon"
+
+// Constructor; uses RAII pattern to be valid after construction
+machmon::machmon() : mach_stats{} {
+  log_init(MONITOR_NAME);
+#undef MONITOR_NAME
+  for (const auto& param : params) {
+    mach_stats.emplace(param.get_name(), prmon::monitored_value(param, true));
+  }
+
+  valid = true;
+}
+
+// Read the first line of /proc/stat and return {utime, stime, ntime}
+
+static std::tuple<long, long, long> read_proc_stat(const std::string read_path = "") {
+  const std::string stat_path = read_path + "/proc/stat";
+  std::ifstream proc_stat{stat_path};
+  if (!proc_stat) {
+    throw std::runtime_error("Cannot open " + stat_path);
+  }
+
+  std::string cpu_label{};
+  long user{}, nice{}, system{};
+
+  // First line: "cpu  user nice system idle ..."
+  if (!(proc_stat >> cpu_label >> user >> nice >> system)) {
+    throw std::runtime_error("Unexpected format in " + stat_path);
+  }
+
+  return std::make_tuple(user, system, nice);
+}
+
+void machmon::update_stats(const std::vector<pid_t>& /*pids*/,
+                              const std::string read_path) {
+    if (!valid) {
+    return;
+    }
+
+    long utime, stime, ntime;
+    std::tie(utime, stime, ntime) = read_proc_stat(read_path);
+
+    // Subtract initial snapshot on first read
+    if (!initialised) {
+      initial_utime = utime;
+      initial_stime = stime;
+      initial_ntime = ntime;
+      initialised = true;
+    }
+
+    const long clk = sysconf(_SC_CLK_TCK);
+    mach_stats.at("utime_total").set_value(std::max(0L, utime - initial_utime) / clk);
+    mach_stats.at("stime_total").set_value(std::max(0L, stime - initial_stime) / clk);
+    mach_stats.at("ntime_total").set_value(std::max(0L, ntime - initial_ntime) / clk);
+}
+
+// Return the summed counters
+prmon::monitored_value_map const machmon::get_text_stats() {
+  prmon::monitored_value_map cpu_stat_map{};
+  for (const auto& value : mach_stats) {
+    cpu_stat_map[value.first] = value.second.get_value();
+  }
+  return cpu_stat_map;
+}
+
+// Same for JSON
+prmon::monitored_value_map const machmon::get_json_total_stats() {
+  return machmon::get_text_stats();
+}
+
+// No averages for global CPU time either
+prmon::monitored_average_map const machmon::get_json_average_stats(
+    unsigned long long /*elapsed_clock_ticks*/) {
+  static const prmon::monitored_average_map empty_average_stats{};
+  return empty_average_stats;
+}
+
+// Return the parameter list
+prmon::parameter_list const machmon::get_parameter_list() { return params; }
+
+// No extra hardware information needed here
+void const machmon::get_hardware_info(nlohmann::json& /*hw_json*/) {
+  return;
+}
+
+void const machmon::get_unit_info(nlohmann::json& unit_json) {
+  prmon::fill_units(unit_json, params);
+  return;
+}