feat(memory): apply kernel memory tunables before memory-mode runs

src/cli/exec/mod.rs

@@ -92,6 +92,7 @@ fn build_orchestrator_config(
         fair_sched: args.shared.experimental.experimental_fair_sched,
         cycle_estimation: args.shared.cycle_estimation,
         exclude_allocations: args.shared.exclude_allocations,
+        memory_tunables: args.shared.memory_tunables,
     })
 }
 

src/cli/run/mod.rs

@@ -50,6 +50,7 @@ impl RunArgs {
         use super::experimental::ExperimentalArgs;
         use super::{PerfRunArgs, ProfilerRunArgs};
         use crate::RunnerMode;
+        use crate::executor::config::MemoryTunables;
 
         Self {
             shared: ExecAndRunSharedArgs {
@@ -71,6 +72,7 @@ impl RunArgs {
                 base: None,
                 cycle_estimation: true,
                 exclude_allocations: false,
+                memory_tunables: MemoryTunables::Auto,
                 profiler_run_args: ProfilerRunArgs {
                     enable_profiler: false,
                     enable_perf: None,
@@ -133,6 +135,7 @@ fn build_orchestrator_config(
         fair_sched: args.shared.experimental.experimental_fair_sched,
         cycle_estimation: args.shared.cycle_estimation,
         exclude_allocations: args.shared.exclude_allocations,
+        memory_tunables: args.shared.memory_tunables,
     })
 }
 

src/cli/shared.rs

@@ -1,6 +1,6 @@
 use super::experimental::ExperimentalArgs;
 use crate::VERSION;
-use crate::executor::config::{SimulationTool, WalltimeProfiler};
+use crate::executor::config::{MemoryTunables, SimulationTool, WalltimeProfiler};
 use crate::prelude::*;
 use crate::run_environment::interfaces::RepositoryProvider;
 use crate::runner_mode::{RunnerMode, load_shell_session_mode};
@@ -135,6 +135,18 @@ pub struct ExecAndRunSharedArgs {
     )]
     pub exclude_allocations: bool,
 
+    /// Stabilise kernel memory behaviour before memory-mode runs: disable THP,
+    /// set vm.compaction_proactiveness=0 / vm.swappiness=0 / kernel.numa_balancing=0,
+    /// disable swap and drop the page cache. `auto` only does this in CI. The
+    /// changes are not reverted afterwards.
+    #[arg(
+        long,
+        value_enum,
+        env = "CODSPEED_MEMORY_TUNABLES",
+        default_value_t = MemoryTunables::Auto
+    )]
+    pub memory_tunables: MemoryTunables,
+
     #[command(flatten)]
     pub profiler_run_args: ProfilerRunArgs,
 

src/executor/config.rs

@@ -51,6 +51,17 @@ pub enum WalltimeProfiler {
     Samply,
 }
 
+/// Whether to apply the kernel memory tunables before a memory-mode run.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)]
+#[value(rename_all = "lower")]
+pub enum MemoryTunables {
+    /// Apply them only when running inside a CI environment.
+    #[default]
+    Auto,
+    On,
+    Off,
+}
+
 /// Run-level configuration owned by the orchestrator.
 ///
 /// Holds all parameters that are constant across benchmark targets within a run,
@@ -95,6 +106,8 @@ pub struct OrchestratorConfig {
     pub cycle_estimation: bool,
     /// Signal the backend to exclude memory allocation time from simulation results.
     pub exclude_allocations: bool,
+    /// Whether to apply kernel memory tunables before memory-mode runs.
+    pub memory_tunables: MemoryTunables,
 }
 
 /// Per-execution configuration passed to executors.
@@ -132,6 +145,8 @@ pub struct ExecutorConfig {
     pub cycle_estimation: bool,
     /// Signal the backend to exclude memory allocation time from simulation results.
     pub exclude_allocations: bool,
+    /// Whether to apply kernel memory tunables before memory-mode runs.
+    pub memory_tunables: MemoryTunables,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -203,6 +218,7 @@ impl OrchestratorConfig {
             fair_sched: self.fair_sched,
             cycle_estimation: self.cycle_estimation,
             exclude_allocations: self.exclude_allocations,
+            memory_tunables: self.memory_tunables,
         }
     }
 }
@@ -237,6 +253,7 @@ impl OrchestratorConfig {
             fair_sched: false,
             cycle_estimation: true,
             exclude_allocations: false,
+            memory_tunables: MemoryTunables::Auto,
         }
     }
 }

src/executor/wall_time/profiler/linux_sysctl.rs → src/executor/helpers/linux_sysctl.rs

@@ -18,7 +18,7 @@ pub fn ensure_linux_profiling_sysctls() -> Result<()> {
 }
 
 #[cfg(target_os = "linux")]
-fn ensure_sysctl(name: &str, target_value: i64) -> Result<()> {
+pub(crate) fn ensure_sysctl(name: &str, target_value: i64) -> Result<()> {
     if sysctl_read(name)? == target_value {
         return Ok(());
     }

src/executor/helpers/mod.rs

@@ -10,6 +10,7 @@ pub mod harvest_perf_maps_for_pids;
 pub mod homebrew;
 pub mod introspected_golang;
 pub mod introspected_nodejs;
+pub mod linux_sysctl;
 pub mod profile_folder;
 pub mod run_command_with_log_pipe;
 pub mod run_with_env;

src/executor/memory/executor.rs

@@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command;
 use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback;
 use crate::executor::helpers::run_with_env::prefix_command_with_env;
 use crate::executor::helpers::run_with_sudo::is_root_user;
+use crate::executor::memory::tunables::apply_memory_tunables;
 use crate::executor::shared::fifo::RunnerFifo;
 use crate::executor::{ExecutionContext, Executor};
 use crate::instruments::mongo_tracer::MongoTracer;
@@ -159,6 +160,10 @@ impl Executor for MemoryExecutor {
         execution_context: &ExecutionContext,
         _mongo_tracer: &Option<MongoTracer>,
     ) -> Result<()> {
+        // Per-run, not in `setup()`: the page cache must be dropped before every
+        // suite, and `setup()` is skipped under --skip-setup.
+        apply_memory_tunables(execution_context.config.memory_tunables);
+
         // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions
         std::fs::create_dir_all(execution_context.profile_folder.join("results"))?;
 

src/executor/memory/mod.rs

@@ -1,2 +1,3 @@
 pub mod executor;
 pub(crate) mod setup;
+pub(crate) mod tunables;

src/executor/memory/tunables.rs

@@ -0,0 +1,221 @@
+//! Kernel knobs that stabilise memory measurements: transparent huge pages,
+//! compaction/swap/NUMA-balancing sysctls, swap and the page cache.
+//!
+//! None of them are reverted afterwards, so they are only applied automatically
+//! on CI runners (see [`MemoryTunables`]).
+
+use crate::executor::config::MemoryTunables;
+use crate::executor::helpers::linux_sysctl::ensure_sysctl;
+use crate::executor::helpers::run_with_sudo::{can_elevate_without_prompt, run_with_sudo};
+use crate::prelude::*;
+use std::fs::read_to_string;
+use std::path::Path;
+
+const THP_DIR: &str = "/sys/kernel/mm/transparent_hugepage";
+const NUMA_BALANCING_PATH: &str = "/proc/sys/kernel/numa_balancing";
+
+/// Applies the knobs on a best-effort basis: a knob that cannot be set is
+/// warned about, never fatal.
+pub fn apply_memory_tunables(setting: MemoryTunables) {
+    match setting {
+        MemoryTunables::Off => {
+            debug!("Kernel memory tunables disabled");
+            return;
+        }
+        MemoryTunables::Auto if !crate::run_environment::is_ci_environment() => {
+            info!(
+                "Not running in CI, skipping kernel memory tunables (pass --memory-tunables=on to force)"
+            );
+            return;
+        }
+        _ => {}
+    }
+
+    // The knobs are an optimisation: measuring with THP on beats blocking the
+    // run on an interactive password prompt.
+    if !can_elevate_without_prompt() {
+        warn!(
+            "Cannot elevate privileges without a password prompt, skipping kernel memory tunables"
+        );
+        return;
+    }
+
+    start_group!("Applying kernel memory tunables");
+    disable_transparent_hugepages();
+    apply_sysctls();
+    disable_swap();
+    drop_page_cache();
+    end_group!();
+}
+
+fn disable_transparent_hugepages() {
+    for knob in ["enabled", "defrag"] {
+        let path = format!("{THP_DIR}/{knob}");
+        // An unprivileged writability test is always false on these root-owned
+        // /sys nodes, so only existence tells us whether the knob is there.
+        if !Path::new(&path).exists() {
+            debug!("{path} does not exist, skipping");
+            continue;
+        }
+        if let Err(error) = write_root_file(&path, "never") {
+            warn!("Failed to disable transparent huge pages ({knob}): {error}");
+        }
+    }
+}
+
+fn apply_sysctls() {
+    let mut knobs = vec![("vm.compaction_proactiveness", 0), ("vm.swappiness", 0)];
+    // Absent on single-node hosts.
+    if Path::new(NUMA_BALANCING_PATH).exists() {
+        knobs.push(("kernel.numa_balancing", 0));
+    }
+
+    for (name, value) in knobs {
+        if let Err(error) = ensure_sysctl(name, value) {
+            warn!("Failed to set {name}={value}: {error}");
+        }
+    }
+}
+
+fn disable_swap() {
+    let decision = match (
+        read_to_string("/proc/swaps"),
+        read_to_string("/proc/meminfo"),
+    ) {
+        (Ok(swaps), Ok(meminfo)) => swapoff_decision(&swaps, &meminfo),
+        _ => SwapoffDecision::Skip("could not read /proc/swaps or /proc/meminfo".to_string()),
+    };
+
+    match decision {
+        SwapoffDecision::Skip(reason) => debug!("Leaving swap enabled: {reason}"),
+        SwapoffDecision::Proceed => {
+            if let Err(error) = run_with_sudo("swapoff", ["-a"]) {
+                warn!("Failed to disable swap: {error}");
+            }
+        }
+    }
+}
+
+fn drop_page_cache() {
+    nix::unistd::sync();
+    if let Err(error) = write_root_file("/proc/sys/vm/drop_caches", "3") {
+        warn!("Failed to drop the page cache: {error}");
+    }
+}
+
+/// Write to a root-owned /proc or /sys node. `sudo tee` needs stdin, which
+/// `run_with_sudo` does not wire up, so redirect inside a shell instead.
+fn write_root_file(path: &str, value: &str) -> Result<()> {
+    run_with_sudo("sh", ["-c", &format!("printf '%s' {value} > {path}")])
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum SwapoffDecision {
+    Proceed,
+    Skip(String),
+}
+
+/// `swapoff -a` faults every swapped page back into RAM and permanently breaks a
+/// zram device (its `disksize` resets to 0 and a later `swapon` fails), so it is
+/// only safe on a plain swap file/partition whose used pages fit in free memory.
+///
+/// `/proc/swaps` columns: `Filename Type Size Used Priority`, first line is a header.
+fn swapoff_decision(swaps: &str, meminfo: &str) -> SwapoffDecision {
+    let skip = |reason: &str| SwapoffDecision::Skip(reason.to_string());
+
+    let rows: Vec<&str> = swaps
+        .lines()
+        .skip(1)
+        .filter(|l| !l.trim().is_empty())
+        .collect();
+    if rows.is_empty() {
+        return skip("no swap is active");
+    }
+
+    let mut used_kib: u64 = 0;
+    for row in rows {
+        let fields: Vec<&str> = row.split_whitespace().collect();
+        let (Some(filename), Some(used)) = (fields.first(), fields.get(3)) else {
+            return skip("could not determine swap usage");
+        };
+        if filename.contains("zram") {
+            return skip("zram swap device present");
+        }
+        let Ok(used) = used.parse::<u64>() else {
+            return skip("could not determine swap usage");
+        };
+        used_kib += used;
+    }
+
+    let Some(available_kib) = parse_mem_available_kib(meminfo) else {
+        return skip("could not determine swap usage");
+    };
+
+    if used_kib >= available_kib {
+        return skip("swapped pages do not fit in available memory");
+    }
+
+    SwapoffDecision::Proceed
+}
+
+fn parse_mem_available_kib(meminfo: &str) -> Option<u64> {
+    meminfo
+        .lines()
+        .find_map(|line| line.strip_prefix("MemAvailable:"))?
+        .split_whitespace()
+        .next()?
+        .parse()
+        .ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const MEMINFO_8G: &str = "MemTotal:       16000000 kB\nMemAvailable:    8000000 kB\n";
+
+    #[test]
+    fn skips_when_no_swap_is_active() {
+        let swaps = "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n";
+        assert_eq!(
+            swapoff_decision(swaps, MEMINFO_8G),
+            SwapoffDecision::Skip("no swap is active".to_string())
+        );
+    }
+
+    #[test]
+    fn skips_zram_swap_devices() {
+        let swaps = "Filename\tType\tSize\tUsed\tPriority\n/dev/zram0 partition 8000000 1024 100\n";
+        assert_eq!(
+            swapoff_decision(swaps, MEMINFO_8G),
+            SwapoffDecision::Skip("zram swap device present".to_string())
+        );
+    }
+
+    #[test]
+    fn skips_when_swapped_pages_do_not_fit_in_memory() {
+        let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file 16000000 8000000 -2\n";
+        assert_eq!(
+            swapoff_decision(swaps, "MemAvailable:    4000000 kB\n"),
+            SwapoffDecision::Skip("swapped pages do not fit in available memory".to_string())
+        );
+    }
+
+    #[test]
+    fn skips_malformed_rows() {
+        let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file\n";
+        assert_eq!(
+            swapoff_decision(swaps, MEMINFO_8G),
+            SwapoffDecision::Skip("could not determine swap usage".to_string())
+        );
+    }
+
+    #[test]
+    fn proceeds_for_a_small_swap_file() {
+        let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file 16000000 1024 -2\n";
+        assert_eq!(
+            swapoff_decision(swaps, MEMINFO_8G),
+            SwapoffDecision::Proceed
+        );
+    }
+}

src/executor/wall_time/profiler/mod.rs

@@ -4,7 +4,6 @@
 //! (perf, samply, instruments, ...) and produces a unified set of artifacts
 //! in the profile folder.
 
-mod linux_sysctl;
 pub mod perf;
 pub mod samply;
 

src/executor/wall_time/profiler/perf/mod.rs

@@ -8,13 +8,13 @@ use crate::executor::helpers::detect_executable::command_has_executable;
 use crate::executor::helpers::env::is_codspeed_debug_enabled;
 use crate::executor::helpers::env::suppress_go_perf_unwinding_warning;
 use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids;
+use crate::executor::helpers::linux_sysctl::ensure_linux_profiling_sysctls;
 use crate::executor::helpers::run_with_sudo::wrap_with_sudo;
 use crate::executor::shared::fifo::FifoBenchmarkData;
 use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING;
 use crate::executor::wall_time::profiler::Profiler;
 use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ;
 use crate::executor::wall_time::profiler::WALLTIME_METADATA_CURRENT_VERSION;
-use crate::executor::wall_time::profiler::linux_sysctl::ensure_linux_profiling_sysctls;
 use crate::executor::wall_time::profiler::perf::perf_executable::get_working_perf_executable;
 use crate::prelude::*;
 use crate::system::SystemInfo;