From 8e5c5f3257bbfee475c38596fe26a21cddf59206 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Tue, 27 Oct 2026 21:32:28 +0000 Subject: [PATCH] fix(supervisor): add configurable log directory VM sandboxes log to their state dir instead of a shared /var/log. Signed-off-by: Eric Curtin --- crates/openshell-core/src/sandbox_env.rs | 6 ++++ crates/openshell-driver-vm/README.md | 2 +- crates/openshell-driver-vm/src/driver.rs | 9 ++++++ .../src/policy_local.rs | 9 +++--- .../src/backend_setup/tests.rs | 2 ++ .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 9 ++++-- crates/openshell-supervisor/src/main.rs | 28 +++++++++++++++++-- crates/openshell-supervisor/src/shared.rs | 2 +- docs/observability/accessing-logs.mdx | 6 ++-- docs/observability/logging.mdx | 4 +-- docs/observability/ocsf-json-export.mdx | 2 +- 12 files changed, 63 insertions(+), 17 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index dea5a9851d..185ef4fee6 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,6 +26,12 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; +/// Directory where the supervisor writes its rotated log files. +pub const LOG_DIR: &str = "OPENSHELL_LOG_DIR"; + +/// Supervisor log directory when [`LOG_DIR`] is not set. +pub const DEFAULT_LOG_DIR: &str = "/var/log"; + /// OTLP/gRPC collector endpoint for supervisor trace export. pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT"; diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index deb4ca9dae..c79424a852 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -284,7 +284,7 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ ``` The VM guest's serial console is appended to `//console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. -The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. +The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Its rotated log files, including OCSF JSONL, go to `logs/` beside them. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. ## Network isolation diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 29930d98bc..73e2c068c4 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -214,6 +214,8 @@ const GUEST_BOUNDARY_CONFIG_DIR: &str = "/.openshell/state"; const GUEST_BOUNDARY_CONFIG_ENV: &str = "OPENSHELL_VM_SANDBOX_BOOTSTRAP"; const HOST_AUTH_BUNDLE_FILE: &str = "supervisor-auth.json"; const HOST_RUNTIME_DESCRIPTOR_FILE: &str = "runtime-descriptor.json"; +/// Private directory in the sandbox state directory for supervisor log files. +const HOST_SUPERVISOR_LOG_DIR: &str = "logs"; const HOST_BOUNDARY_GENERATION_FILE: &str = "boundary-generation"; /// Per-sandbox host-only copy of the gateway-normalized destination trust. /// @@ -1428,6 +1430,12 @@ impl VmDriver { .map_err(|error| Status::internal(format!("encode main process spec: {error}")))?; let upstream_proxy_args = upstream_proxy_cli_args(&self.config) .map_err(|error| Status::invalid_argument(format!("render upstream proxy: {error}")))?; + // The host supervisor cannot count on a writable `/var/log`, and + // sandboxes must not share log files there. + let log_dir = state_dir.join(HOST_SUPERVISOR_LOG_DIR); + create_private_dir_all(&log_dir) + .await + .map_err(|error| Status::internal(format!("create supervisor log dir: {error}")))?; let mut command = Command::new(&supervisor_binary); isolate_host_control_environment(&mut command); command @@ -1482,6 +1490,7 @@ impl VmDriver { openshell_core::sandbox_env::LOG_LEVEL, openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level), ) + .env(openshell_core::sandbox_env::LOG_DIR, &log_dir) .env( openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index 033f492aa7..e220367ede 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -76,7 +76,6 @@ const MAX_DENIALS_LIMIT: usize = 100; /// most recent two so a request just past midnight still has yesterday's /// denials. const DENIAL_LOG_FILES_TO_SCAN: usize = 2; -const LOG_DIR: &str = "/var/log"; /// `/v1/denials` reads `openshell.YYYY-MM-DD.log`, not the OCSF `.jsonl` files, /// which are opt-in via `ocsf_json_enabled`. const SHORTHAND_LOG_PREFIX: &str = "openshell."; @@ -100,7 +99,7 @@ pub enum DenialLogSource { impl Default for DenialLogSource { fn default() -> Self { - Self::ShorthandFiles(PathBuf::from(LOG_DIR)) + Self::ShorthandFiles(PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR)) } } @@ -441,7 +440,7 @@ async fn recent_denials_response( // `[]` in both cases and cannot tell the difference. The shorthand log is // always-on (no setting gates it), so the only way `log_available=false` // happens in practice is if the supervisor has not flushed any events to - // disk yet, or `/var/log` is not writable in this image. + // disk yet, or the log directory is not writable. let log_available = matches!( collect_shorthand_log_files(&log_dir, 1), Ok(files) if !files.is_empty() @@ -457,7 +456,7 @@ async fn recent_denials_response( }); if !log_available { payload["note"] = serde_json::json!( - "no shorthand log file is present yet at /var/log/openshell.YYYY-MM-DD.log; the supervisor may not have emitted any events to disk yet" + "no shorthand log file is present yet; the supervisor may not have emitted any events to disk yet" ); } @@ -1813,7 +1812,7 @@ mod tests { payload["note"] .as_str() .unwrap() - .contains("/var/log/openshell.") + .contains("no shorthand log file") ); } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 56ff9b5052..9a8d6c2895 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -854,6 +854,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { &config, openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); Box::pin(crate::run_sandbox_with_backend( &setup, @@ -1057,6 +1058,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { &config, openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); Box::pin(crate::run_sandbox_with_backend( &setup, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 8e6def6552..95e3d5bed9 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -312,6 +312,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) .unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + std::path::PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); let startup = run_sandbox_with_backend(&setup, config, bindings, None, None); tokio::pin!(startup); diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 07614d7474..cf45ef2e0a 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -953,19 +953,23 @@ pub struct SandboxInstanceBindings { impl SandboxInstanceBindings { /// Bindings of a process dedicated to the instance `config` launches. Its - /// process-wide log push and shorthand log files serve the instance. + /// process-wide log push and the shorthand log files in `log_dir` serve + /// the instance. #[must_use] pub fn dedicated( config: &SandboxInstanceConfig, gateway_credentials: openshell_core::grpc_client::GatewayCredentials, ocsf_json: openshell_ocsf::OcsfJsonSettings, + log_dir: std::path::PathBuf, ) -> Self { Self { gateway_credentials, event_context: Arc::new(config.event_context()), ocsf_json, log_sink: None, - denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::ShorthandFiles( + log_dir, + ), } } } @@ -4615,6 +4619,7 @@ mod tests { openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) .expect("credentials"), openshell_ocsf::OcsfJsonSettings::default(), + std::path::PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); let error = run_sandbox_with_backend( &backend_setup::OpenShellBackendSetup, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 9a368110ea..c19b4bb2c5 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -134,6 +134,14 @@ struct Args { #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] log_level: String, + /// Directory for rotated supervisor log files. + #[arg( + long, + default_value = openshell_core::sandbox_env::DEFAULT_LOG_DIR, + env = openshell_core::sandbox_env::LOG_DIR + )] + log_dir: PathBuf, + /// OTLP/gRPC collector endpoint for supervisor trace export. #[arg(long, env = openshell_core::sandbox_env::OTLP_ENDPOINT)] otlp_endpoint: Option, @@ -526,7 +534,7 @@ fn main() -> Result<()> { let (logging_layers, _logging_guards, file_available) = logging::layers( std::io::stderr(), - Path::new("/var/log"), + &args.log_dir, console_filter, ocsf_enabled.clone(), ocsf_schema_version.clone(), @@ -543,7 +551,10 @@ fn main() -> Result<()> { ) .init(); if !file_available { - warn!("Could not open /var/log for log rotation; using stderr-only logging"); + warn!( + log_dir = %args.log_dir.display(), + "Could not open log directory for rotation; using stderr-only logging" + ); } if let Some(error) = otlp_setup_error { warn!(%error, "OTLP exporting could not be started; continuing without it"); @@ -638,6 +649,7 @@ fn main() -> Result<()> { enabled: ocsf_enabled, target_version: ocsf_schema_version, }, + args.log_dir.clone(), ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( backend_setup.as_ref(), @@ -987,6 +999,18 @@ mod tests { ); } + #[test] + fn log_dir_defaults_to_var_log_and_accepts_an_override() { + let args = Args::try_parse_from(["openshell-supervisor"]).unwrap(); + assert_eq!( + args.log_dir, + Path::new(openshell_core::sandbox_env::DEFAULT_LOG_DIR) + ); + let args = + Args::try_parse_from(["openshell-supervisor", "--log-dir", "/tmp/logs"]).unwrap(); + assert_eq!(args.log_dir, Path::new("/tmp/logs")); + } + #[test] fn isolation_backend_inputs_are_mandatory() { let args = Args::try_parse_from(["openshell-supervisor"]).expect("parse defaults"); diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 2f449327ea..42e91809b1 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -91,7 +91,7 @@ from sandbox runtimes)", // settings apply to the events its threads emit. let (logging_layers, _logging_guards, _) = crate::logging::layers( std::io::stderr(), - std::path::Path::new("/var/log"), + &args.log_dir, console_filter, Arc::new(AtomicBool::new(false)), Arc::new(std::sync::Mutex::new(String::new())), diff --git a/docs/observability/accessing-logs.mdx b/docs/observability/accessing-logs.mdx index 9e19c637bf..380a24218d 100644 --- a/docs/observability/accessing-logs.mdx +++ b/docs/observability/accessing-logs.mdx @@ -78,13 +78,13 @@ The gateway merges the log and platform event sources before emitting, so events ## Direct Filesystem Access -The supervisor writes shorthand logs to `/var/log/openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in its own filesystem. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. +The supervisor writes shorthand logs to `openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `openshell-ocsf.YYYY-MM-DD.jsonl` in its log directory, `/var/log` unless the driver sets another. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. | Compute driver | Supervisor log location | Local storage lifetime | |---|---|---| | Kubernetes | `/var/log` in the separate supervisor pod's `supervisor` container | An `emptyDir` volume, removed with the pod. | | Docker / Podman | `/var/log` in the separate supervisor container | A tmpfs mount, lost when the container stops. | -| VM | `/var/log` on the host running the supervisor, if writable | Host-managed storage. The driver also captures stderr in `supervisor.err.log` in the sandbox's driver state directory. | +| VM | `logs/` in the sandbox's driver state directory on the host | Removed with the sandbox. The driver also captures stderr in `supervisor.err.log` beside it. | | Windows MXC | Gateway-local JSONL output | See [Windows MXC gateways](/observability/ocsf-json-export#windows-mxc-gateways). | Use platform tooling with access to the supervisor resources. For Podman, copy the current supervisor files to a local directory while the container is running: @@ -108,7 +108,7 @@ Shorthand and diagnostics follow the supervisor's diagnostic log filter. With `o The default supervisor image has no shell or `tar`, so shell-based `kubectl exec` and `kubectl cp` examples do not work with that image. Reading Kubernetes supervisor files requires operator-managed access to its log volume, such as cluster debugging tooling. OpenShell does not configure a file collector for you. -Ship files before rotation or resource teardown and configure your collector to handle daily rotation. Local file writers are non-blocking and can also drop events under pressure. If the supervisor cannot open `/var/log`, it falls back to stderr-only logging and does not produce these files, even when `ocsf_json_enabled` is true. Neither the local files nor the gateway stream guarantees complete audit history. +Ship files before rotation or resource teardown and configure your collector to handle daily rotation. Local file writers are non-blocking and can also drop events under pressure. If the supervisor cannot open its log directory, it falls back to stderr-only logging and does not produce these files, even when `ocsf_json_enabled` is true. Neither the local files nor the gateway stream guarantees complete audit history. ## Filtering by Event Type diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index d7f068dca9..a95c1b248f 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -280,7 +280,7 @@ Set `landlock.compatibility` to `hard_requirement` in the policy to make these f ## Log File Location -The supervisor writes logs to `/var/log/` in its own filesystem, separate from the sandbox workload: +The supervisor writes logs to its log directory, `/var/log/` unless the driver sets another, in its own filesystem, separate from the sandbox workload: | File | Format | Rotation | |---|---|---| @@ -292,7 +292,7 @@ Each format rotates daily with an independent three-file limit. Configure supervisor OCSF collectors to match `openshell-ocsf.*.jsonl`. Shorthand filenames and Windows MXC gateway filenames are unchanged. -File output requires a writable `/var/log` and uses non-blocking writers that can drop events under pressure. If file logging cannot start, the supervisor falls back to stderr-only output. Enabling `ocsf_json_enabled` adds full `OCSF-JSON` records to stderr even without a file sink; see [OCSF JSON collection](/observability/ocsf-json-export#output-location). +File output requires a writable log directory and uses non-blocking writers that can drop events under pressure. If file logging cannot start, the supervisor falls back to stderr-only output. Enabling `ocsf_json_enabled` adds full `OCSF-JSON` records to stderr even without a file sink; see [OCSF JSON collection](/observability/ocsf-json-export#output-location). Docker and Podman use tmpfs for this directory; Kubernetes uses an `emptyDir` in the supervisor pod. Collect logs externally before rotation or teardown for durable retention. See [Accessing Logs](/observability/accessing-logs#direct-filesystem-access) for driver-specific access and storage lifetimes. diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 96e30c75c0..ad9c3efa4e 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -131,7 +131,7 @@ kubectl -n logs -c supervisor --follow JSON stderr output works independently of writable log storage and the diagnostic log filter. Disabling the setting stops new JSON records. The console writer uses a bounded queue of 1,024 complete lines shared by diagnostics, shorthand, and JSON; when the queue is full, new lines are dropped rather than blocking enforcement. Delivery is best-effort, and retention depends on your runtime and external collector. The gateway log API and `openshell logs` continue to return shorthand. -When file logging is available, supervisors also write plain JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable `/var/log`; a missing file sink does not disable JSON stderr output. +When file logging is available, supervisors also write plain JSONL to `openshell-ocsf.YYYY-MM-DD.jsonl` in their log directory (`/var/log` unless the driver sets another), in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable log directory; a missing file sink does not disable JSON stderr output. This file contains supervisor-originated events, not the [sandbox-runtime events](/observability/accessing-logs#main-process-output) emitted into workload container logs.