diff --git a/src/cli/exec/mod.rs b/src/cli/exec/mod.rs index 5ae0692af..4ca57d5f2 100644 --- a/src/cli/exec/mod.rs +++ b/src/cli/exec/mod.rs @@ -92,6 +92,7 @@ fn build_orchestrator_config( fair_sched: args.shared.experimental.experimental_fair_sched, cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, + memory_tunables: args.shared.memory_tunables, }) } diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index e5444d49b..33c9d39bc 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -50,6 +50,7 @@ impl RunArgs { use super::experimental::ExperimentalArgs; use super::{PerfRunArgs, ProfilerRunArgs}; use crate::RunnerMode; + use crate::executor::config::MemoryTunables; Self { shared: ExecAndRunSharedArgs { @@ -71,6 +72,7 @@ impl RunArgs { base: None, cycle_estimation: true, exclude_allocations: false, + memory_tunables: MemoryTunables::Auto, profiler_run_args: ProfilerRunArgs { enable_profiler: false, enable_perf: None, @@ -133,6 +135,7 @@ fn build_orchestrator_config( fair_sched: args.shared.experimental.experimental_fair_sched, cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, + memory_tunables: args.shared.memory_tunables, }) } diff --git a/src/cli/shared.rs b/src/cli/shared.rs index 3767bbf35..a4e2ae73f 100644 --- a/src/cli/shared.rs +++ b/src/cli/shared.rs @@ -1,6 +1,6 @@ use super::experimental::ExperimentalArgs; use crate::VERSION; -use crate::executor::config::{SimulationTool, WalltimeProfiler}; +use crate::executor::config::{MemoryTunables, SimulationTool, WalltimeProfiler}; use crate::prelude::*; use crate::run_environment::interfaces::RepositoryProvider; use crate::runner_mode::{RunnerMode, load_shell_session_mode}; @@ -135,6 +135,18 @@ pub struct ExecAndRunSharedArgs { )] pub exclude_allocations: bool, + /// Stabilise kernel memory behaviour before memory-mode runs: disable THP, + /// set vm.compaction_proactiveness=0 / vm.swappiness=0 / kernel.numa_balancing=0, + /// disable swap and drop the page cache. `auto` only does this in CI. The + /// changes are not reverted afterwards. + #[arg( + long, + value_enum, + env = "CODSPEED_MEMORY_TUNABLES", + default_value_t = MemoryTunables::Auto + )] + pub memory_tunables: MemoryTunables, + #[command(flatten)] pub profiler_run_args: ProfilerRunArgs, diff --git a/src/executor/config.rs b/src/executor/config.rs index 0f7dfe24a..ef1112ad3 100644 --- a/src/executor/config.rs +++ b/src/executor/config.rs @@ -51,6 +51,17 @@ pub enum WalltimeProfiler { Samply, } +/// Whether to apply the kernel memory tunables before a memory-mode run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)] +#[value(rename_all = "lower")] +pub enum MemoryTunables { + /// Apply them only when running inside a CI environment. + #[default] + Auto, + On, + Off, +} + /// Run-level configuration owned by the orchestrator. /// /// Holds all parameters that are constant across benchmark targets within a run, @@ -95,6 +106,8 @@ pub struct OrchestratorConfig { pub cycle_estimation: bool, /// Signal the backend to exclude memory allocation time from simulation results. pub exclude_allocations: bool, + /// Whether to apply kernel memory tunables before memory-mode runs. + pub memory_tunables: MemoryTunables, } /// Per-execution configuration passed to executors. @@ -132,6 +145,8 @@ pub struct ExecutorConfig { pub cycle_estimation: bool, /// Signal the backend to exclude memory allocation time from simulation results. pub exclude_allocations: bool, + /// Whether to apply kernel memory tunables before memory-mode runs. + pub memory_tunables: MemoryTunables, } #[derive(Debug, Clone, PartialEq)] @@ -203,6 +218,7 @@ impl OrchestratorConfig { fair_sched: self.fair_sched, cycle_estimation: self.cycle_estimation, exclude_allocations: self.exclude_allocations, + memory_tunables: self.memory_tunables, } } } @@ -237,6 +253,7 @@ impl OrchestratorConfig { fair_sched: false, cycle_estimation: true, exclude_allocations: false, + memory_tunables: MemoryTunables::Auto, } } } diff --git a/src/executor/wall_time/profiler/linux_sysctl.rs b/src/executor/helpers/linux_sysctl.rs similarity index 95% rename from src/executor/wall_time/profiler/linux_sysctl.rs rename to src/executor/helpers/linux_sysctl.rs index 657581ecb..98ddac71e 100644 --- a/src/executor/wall_time/profiler/linux_sysctl.rs +++ b/src/executor/helpers/linux_sysctl.rs @@ -18,7 +18,7 @@ pub fn ensure_linux_profiling_sysctls() -> Result<()> { } #[cfg(target_os = "linux")] -fn ensure_sysctl(name: &str, target_value: i64) -> Result<()> { +pub(crate) fn ensure_sysctl(name: &str, target_value: i64) -> Result<()> { if sysctl_read(name)? == target_value { return Ok(()); } diff --git a/src/executor/helpers/mod.rs b/src/executor/helpers/mod.rs index 6efbf5cc8..5318ccbaa 100644 --- a/src/executor/helpers/mod.rs +++ b/src/executor/helpers/mod.rs @@ -10,6 +10,7 @@ pub mod harvest_perf_maps_for_pids; pub mod homebrew; pub mod introspected_golang; pub mod introspected_nodejs; +pub mod linux_sysctl; pub mod profile_folder; pub mod run_command_with_log_pipe; pub mod run_with_env; diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index e12625646..8120814a3 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::tunables::apply_memory_tunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; use crate::instruments::mongo_tracer::MongoTracer; @@ -159,6 +160,10 @@ impl Executor for MemoryExecutor { execution_context: &ExecutionContext, _mongo_tracer: &Option, ) -> Result<()> { + // Per-run, not in `setup()`: the page cache must be dropped before every + // suite, and `setup()` is skipped under --skip-setup. + apply_memory_tunables(execution_context.config.memory_tunables); + // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index e0ac4745c..2d17547d1 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,2 +1,3 @@ pub mod executor; pub(crate) mod setup; +pub(crate) mod tunables; diff --git a/src/executor/memory/tunables.rs b/src/executor/memory/tunables.rs new file mode 100644 index 000000000..b565867ed --- /dev/null +++ b/src/executor/memory/tunables.rs @@ -0,0 +1,221 @@ +//! Kernel knobs that stabilise memory measurements: transparent huge pages, +//! compaction/swap/NUMA-balancing sysctls, swap and the page cache. +//! +//! None of them are reverted afterwards, so they are only applied automatically +//! on CI runners (see [`MemoryTunables`]). + +use crate::executor::config::MemoryTunables; +use crate::executor::helpers::linux_sysctl::ensure_sysctl; +use crate::executor::helpers::run_with_sudo::{can_elevate_without_prompt, run_with_sudo}; +use crate::prelude::*; +use std::fs::read_to_string; +use std::path::Path; + +const THP_DIR: &str = "/sys/kernel/mm/transparent_hugepage"; +const NUMA_BALANCING_PATH: &str = "/proc/sys/kernel/numa_balancing"; + +/// Applies the knobs on a best-effort basis: a knob that cannot be set is +/// warned about, never fatal. +pub fn apply_memory_tunables(setting: MemoryTunables) { + match setting { + MemoryTunables::Off => { + debug!("Kernel memory tunables disabled"); + return; + } + MemoryTunables::Auto if !crate::run_environment::is_ci_environment() => { + info!( + "Not running in CI, skipping kernel memory tunables (pass --memory-tunables=on to force)" + ); + return; + } + _ => {} + } + + // The knobs are an optimisation: measuring with THP on beats blocking the + // run on an interactive password prompt. + if !can_elevate_without_prompt() { + warn!( + "Cannot elevate privileges without a password prompt, skipping kernel memory tunables" + ); + return; + } + + start_group!("Applying kernel memory tunables"); + disable_transparent_hugepages(); + apply_sysctls(); + disable_swap(); + drop_page_cache(); + end_group!(); +} + +fn disable_transparent_hugepages() { + for knob in ["enabled", "defrag"] { + let path = format!("{THP_DIR}/{knob}"); + // An unprivileged writability test is always false on these root-owned + // /sys nodes, so only existence tells us whether the knob is there. + if !Path::new(&path).exists() { + debug!("{path} does not exist, skipping"); + continue; + } + if let Err(error) = write_root_file(&path, "never") { + warn!("Failed to disable transparent huge pages ({knob}): {error}"); + } + } +} + +fn apply_sysctls() { + let mut knobs = vec![("vm.compaction_proactiveness", 0), ("vm.swappiness", 0)]; + // Absent on single-node hosts. + if Path::new(NUMA_BALANCING_PATH).exists() { + knobs.push(("kernel.numa_balancing", 0)); + } + + for (name, value) in knobs { + if let Err(error) = ensure_sysctl(name, value) { + warn!("Failed to set {name}={value}: {error}"); + } + } +} + +fn disable_swap() { + let decision = match ( + read_to_string("/proc/swaps"), + read_to_string("/proc/meminfo"), + ) { + (Ok(swaps), Ok(meminfo)) => swapoff_decision(&swaps, &meminfo), + _ => SwapoffDecision::Skip("could not read /proc/swaps or /proc/meminfo".to_string()), + }; + + match decision { + SwapoffDecision::Skip(reason) => debug!("Leaving swap enabled: {reason}"), + SwapoffDecision::Proceed => { + if let Err(error) = run_with_sudo("swapoff", ["-a"]) { + warn!("Failed to disable swap: {error}"); + } + } + } +} + +fn drop_page_cache() { + nix::unistd::sync(); + if let Err(error) = write_root_file("/proc/sys/vm/drop_caches", "3") { + warn!("Failed to drop the page cache: {error}"); + } +} + +/// Write to a root-owned /proc or /sys node. `sudo tee` needs stdin, which +/// `run_with_sudo` does not wire up, so redirect inside a shell instead. +fn write_root_file(path: &str, value: &str) -> Result<()> { + run_with_sudo("sh", ["-c", &format!("printf '%s' {value} > {path}")]) +} + +#[derive(Debug, PartialEq, Eq)] +enum SwapoffDecision { + Proceed, + Skip(String), +} + +/// `swapoff -a` faults every swapped page back into RAM and permanently breaks a +/// zram device (its `disksize` resets to 0 and a later `swapon` fails), so it is +/// only safe on a plain swap file/partition whose used pages fit in free memory. +/// +/// `/proc/swaps` columns: `Filename Type Size Used Priority`, first line is a header. +fn swapoff_decision(swaps: &str, meminfo: &str) -> SwapoffDecision { + let skip = |reason: &str| SwapoffDecision::Skip(reason.to_string()); + + let rows: Vec<&str> = swaps + .lines() + .skip(1) + .filter(|l| !l.trim().is_empty()) + .collect(); + if rows.is_empty() { + return skip("no swap is active"); + } + + let mut used_kib: u64 = 0; + for row in rows { + let fields: Vec<&str> = row.split_whitespace().collect(); + let (Some(filename), Some(used)) = (fields.first(), fields.get(3)) else { + return skip("could not determine swap usage"); + }; + if filename.contains("zram") { + return skip("zram swap device present"); + } + let Ok(used) = used.parse::() else { + return skip("could not determine swap usage"); + }; + used_kib += used; + } + + let Some(available_kib) = parse_mem_available_kib(meminfo) else { + return skip("could not determine swap usage"); + }; + + if used_kib >= available_kib { + return skip("swapped pages do not fit in available memory"); + } + + SwapoffDecision::Proceed +} + +fn parse_mem_available_kib(meminfo: &str) -> Option { + meminfo + .lines() + .find_map(|line| line.strip_prefix("MemAvailable:"))? + .split_whitespace() + .next()? + .parse() + .ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + const MEMINFO_8G: &str = "MemTotal: 16000000 kB\nMemAvailable: 8000000 kB\n"; + + #[test] + fn skips_when_no_swap_is_active() { + let swaps = "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"; + assert_eq!( + swapoff_decision(swaps, MEMINFO_8G), + SwapoffDecision::Skip("no swap is active".to_string()) + ); + } + + #[test] + fn skips_zram_swap_devices() { + let swaps = "Filename\tType\tSize\tUsed\tPriority\n/dev/zram0 partition 8000000 1024 100\n"; + assert_eq!( + swapoff_decision(swaps, MEMINFO_8G), + SwapoffDecision::Skip("zram swap device present".to_string()) + ); + } + + #[test] + fn skips_when_swapped_pages_do_not_fit_in_memory() { + let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file 16000000 8000000 -2\n"; + assert_eq!( + swapoff_decision(swaps, "MemAvailable: 4000000 kB\n"), + SwapoffDecision::Skip("swapped pages do not fit in available memory".to_string()) + ); + } + + #[test] + fn skips_malformed_rows() { + let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file\n"; + assert_eq!( + swapoff_decision(swaps, MEMINFO_8G), + SwapoffDecision::Skip("could not determine swap usage".to_string()) + ); + } + + #[test] + fn proceeds_for_a_small_swap_file() { + let swaps = "Filename\tType\tSize\tUsed\tPriority\n/swapfile file 16000000 1024 -2\n"; + assert_eq!( + swapoff_decision(swaps, MEMINFO_8G), + SwapoffDecision::Proceed + ); + } +} diff --git a/src/executor/wall_time/profiler/mod.rs b/src/executor/wall_time/profiler/mod.rs index ab7f62cb6..07258986a 100644 --- a/src/executor/wall_time/profiler/mod.rs +++ b/src/executor/wall_time/profiler/mod.rs @@ -4,7 +4,6 @@ //! (perf, samply, instruments, ...) and produces a unified set of artifacts //! in the profile folder. -mod linux_sysctl; pub mod perf; pub mod samply; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 2c5514d24..9d799ffe7 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -8,13 +8,13 @@ use crate::executor::helpers::detect_executable::command_has_executable; use crate::executor::helpers::env::is_codspeed_debug_enabled; use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; +use crate::executor::helpers::linux_sysctl::ensure_linux_profiling_sysctls; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; use crate::executor::wall_time::profiler::WALLTIME_METADATA_CURRENT_VERSION; -use crate::executor::wall_time::profiler::linux_sysctl::ensure_linux_profiling_sysctls; use crate::executor::wall_time::profiler::perf::perf_executable::get_working_perf_executable; use crate::prelude::*; use crate::system::SystemInfo; diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 97b5bffd9..b77209ceb 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -4,10 +4,10 @@ use crate::cli::InternalCommands; use crate::cli::samply::SamplyArgs; use crate::executor::ExecutorConfig; use crate::executor::helpers::command::CommandBuilder; +use crate::executor::helpers::linux_sysctl::ensure_linux_profiling_sysctls; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; use crate::executor::wall_time::profiler::Profiler; -use crate::executor::wall_time::profiler::linux_sysctl::ensure_linux_profiling_sysctls; use crate::prelude::*; use crate::system::SystemInfo; use async_trait::async_trait; diff --git a/src/run_environment/mod.rs b/src/run_environment/mod.rs index 77ba7faa0..0872a3fb8 100644 --- a/src/run_environment/mod.rs +++ b/src/run_environment/mod.rs @@ -53,3 +53,9 @@ pub async fn get_provider( Ok(provider) } + +/// Whether the runner is executing inside a CI environment, i.e. any provider +/// other than the local fallback claims the environment. +pub fn is_ci_environment() -> bool { + BuildkiteProvider::detect() || GitHubActionsProvider::detect() || GitLabCIProvider::detect() +}