diff --git a/Cargo.lock b/Cargo.lock index 8b20796603..626c9324f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4098,6 +4098,7 @@ dependencies = [ "url", "uuid", "windows-sys 0.61.2", + "xxhash-rust", ] [[package]] @@ -6510,6 +6511,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" + [[package]] name = "xz2" version = "0.1.7" diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs index 407823453d..2484267baf 100644 --- a/crates/ov_cli/src/client.rs +++ b/crates/ov_cli/src/client.rs @@ -510,20 +510,6 @@ impl HttpClient { self.post("/api/v1/system/consistency", &body).await } - pub async fn backend_sync_status(&self, uri: &str) -> Result { - let body = serde_json::json!({ - "uri": uri, - }); - self.post("/api/v1/system/backend/sync-status", &body).await - } - - pub async fn backend_sync_retry(&self, uri: &str) -> Result { - let body = serde_json::json!({ - "uri": uri, - }); - self.post("/api/v1/system/backend/sync-retry", &body).await - } - /// Download file as raw bytes pub async fn get_bytes(&self, uri: &str) -> Result> { let url = format!("{}/api/v1/content/download", self.base.base_url); diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rs index 5ad24fff85..037cfae04d 100644 --- a/crates/ov_cli/src/commands/system.rs +++ b/crates/ov_cli/src/commands/system.rs @@ -81,28 +81,6 @@ pub async fn consistency( Ok(()) } -pub async fn backend_sync_status( - client: &HttpClient, - uri: &str, - output_format: OutputFormat, - compact: bool, -) -> Result<()> { - let response: serde_json::Value = client.backend_sync_status(uri).await?; - output_success(&response, output_format, compact); - Ok(()) -} - -pub async fn backend_sync_retry( - client: &HttpClient, - uri: &str, - output_format: OutputFormat, - compact: bool, -) -> Result<()> { - let response: serde_json::Value = client.backend_sync_retry(uri).await?; - output_success(&response, output_format, compact); - Ok(()) -} - fn output_consistency_table(response: &serde_json::Value, compact: bool) { let summary = json!({ "ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false), diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rs index c94397d8f1..45db152909 100644 --- a/crates/ov_cli/src/error_ui.rs +++ b/crates/ov_cli/src/error_ui.rs @@ -529,12 +529,6 @@ fn system_help_command(program: &str, tokens: &[&str]) -> String { Some("init-key") => format!("{program} system crypto init-key --help"), _ => format!("{program} system crypto --help"), }, - Some("backend") => match tokens.get(3).copied() { - Some("sync-status" | "sync-retry") => { - format!("{program} system backend {} --help", tokens[3]) - } - _ => format!("{program} system backend --help"), - }, Some("wait" | "status" | "health" | "consistency") => { format!("{program} system {} --help", tokens[2]) } diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs index 8c927503c6..8bf132c1fd 100644 --- a/crates/ov_cli/src/handlers.rs +++ b/crates/ov_cli/src/handlers.rs @@ -415,7 +415,7 @@ pub async fn handle_restore( .await } -use crate::{SystemBackendCommands, SystemCommands}; +use crate::SystemCommands; pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> { let client = ctx.get_client(); @@ -440,16 +440,6 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> { commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await } SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await, - SystemCommands::Backend { action } => match action { - SystemBackendCommands::SyncStatus { uri } => { - commands::system::backend_sync_status(&client, &uri, ctx.output_format, ctx.compact) - .await - } - SystemBackendCommands::SyncRetry { uri } => { - commands::system::backend_sync_retry(&client, &uri, ctx.output_format, ctx.compact) - .await - } - }, } } diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs index d386936be9..17f4015f13 100644 --- a/crates/ov_cli/src/help_ui.rs +++ b/crates/ov_cli/src/help_ui.rs @@ -1340,7 +1340,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ }, CommandHelpSpec { path: &["system"], - purpose: "Run server utility, health, consistency, backend sync, and crypto commands.", + purpose: "Run server utility, health, consistency, and crypto commands.", examples: &[ HelpItem { label: "ov system health", @@ -1350,34 +1350,12 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[ label: "ov system consistency viking://projects/acme", description: "Check filesystem/vector consistency.", }, - HelpItem { - label: "ov system backend sync-status viking://projects/acme", - description: "Inspect multi-write backend sync lag.", - }, ], next_steps: &[HelpItem { label: "ov status", description: "Use the standard status view.", }], }, - CommandHelpSpec { - path: &["system", "backend"], - purpose: "Inspect and repair multi-write backend sync state.", - examples: &[ - HelpItem { - label: "ov system backend sync-status viking://resources", - description: "Show pending and acknowledged backend sync state.", - }, - HelpItem { - label: "ov system backend sync-retry viking://resources", - description: "Retry lagging backend sync targets.", - }, - ], - next_steps: &[HelpItem { - label: "ov system backend sync-status ", - description: "Inspect the subtree again after retry.", - }], - }, CommandHelpSpec { path: &["reindex"], purpose: "Reindex semantic/vector artifacts for a URI.", @@ -2992,7 +2970,6 @@ mod tests { fn curated_help_lists_timeout_for_waiting_commands() { for args in [ ["ov", "add-resource", "--help"], - ["ov", "add-skill", "--help"], ["ov", "rm", "--help"], ["ov", "write", "--help"], ] { @@ -3214,7 +3191,7 @@ mod tests { ("admin", "Manage accounts, users, roles, and API keys."), ( "system", - "Run server utility, health, consistency, backend sync, and crypto commands.", + "Run server utility, health, consistency, and crypto commands.", ), ("observer", "Inspect specific OpenViking server subsystems."), ] { @@ -3281,20 +3258,6 @@ mod tests { ); } - #[test] - fn supported_nested_curated_help_still_renders() { - let rendered = strip_ansi( - &render_command_help_request(&os_args(&["ov", "system", "backend", "--help"])) - .expect("system backend help should render curated nested help"), - ); - - assert!(rendered.contains("ov system backend")); - assert!(rendered.contains("sync-status")); - assert!(rendered.contains("sync-retry")); - assert!(!rendered.contains("help Print this message")); - assert!(rendered.contains("--sudo")); - } - #[test] fn prefixed_help_with_positional_value_renders_curated_command_help() { let rendered = strip_ansi( diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs index 30ff8c4d6c..00630c3edf 100644 --- a/crates/ov_cli/src/main.rs +++ b/crates/ov_cli/src/main.rs @@ -1589,29 +1589,6 @@ enum SystemCommands { #[command(subcommand)] action: commands::crypto::CryptoCommands, }, - /// Backend sync inspection and repair commands - Backend { - #[command(subcommand)] - action: SystemBackendCommands, - }, -} - -#[derive(Subcommand)] -enum SystemBackendCommands { - /// Show multi-write backend sync status for a URI subtree - #[command(name = "sync-status")] - SyncStatus { - /// Viking URI to inspect - #[arg(value_name = "uri")] - uri: String, - }, - /// Retry pending multi-write backend sync work for a URI subtree - #[command(name = "sync-retry")] - SyncRetry { - /// Viking URI to repair - #[arg(value_name = "uri")] - uri: String, - }, } #[derive(Subcommand)] @@ -2544,14 +2521,6 @@ fn plain_help_misuse(args: &[OsString]) -> Option { { path.push(watch_subcommand.to_string()); } - if command == "system" - && path.get(1).is_some_and(|token| token == "backend") - && let Some(backend_subcommand) = - tokens.get(2).map(|token| canonical_plain_help_token(token)) - && backend_subcommand != "help" - { - path.push(backend_subcommand.to_string()); - } return Some(PlainHelpMisuse { help_command: prefixed_help_command(&path), }); @@ -2763,10 +2732,6 @@ fn known_task_command_requires_config(tokens: &[String]) -> bool { fn known_system_command_requires_config(tokens: &[String]) -> bool { match tokens.get(1).map(String::as_str) { Some("wait" | "status" | "health" | "consistency") => true, - Some("backend") => matches!( - tokens.get(2).map(String::as_str), - None | Some("sync-status" | "sync-retry") - ), Some("crypto") => matches!(tokens.get(2).map(String::as_str), None | Some("init-key")), _ => false, } @@ -4032,7 +3997,7 @@ mod tests { }; use crate::config::{Config, DEFAULT_CUSTOM_URL}; use crate::output::OutputFormat; - use crate::{AdminCommands, SystemBackendCommands, SystemCommands, handlers}; + use crate::{AdminCommands, handlers}; use clap::{CommandFactory, Parser}; use std::ffi::OsString; @@ -4382,22 +4347,6 @@ mod tests { } } - #[test] - fn cli_parses_system_backend_sync_status() { - let cli = Cli::try_parse_from(["ov", "system", "backend", "sync-status", "viking://a"]) - .expect("system backend sync-status should parse"); - - match cli.command { - Commands::System { action } => match action { - SystemCommands::Backend { action } => match action { - SystemBackendCommands::SyncStatus { uri } => assert_eq!(uri, "viking://a"), - _ => panic!("expected backend sync-status command"), - }, - _ => panic!("expected system backend command"), - }, - _ => panic!("expected system command"), - } - } #[test] fn server_commands_require_existing_cli_config() { @@ -4623,7 +4572,6 @@ mod tests { &["ov", "config", "validate"], &["ov", "config", "show"], &["ov", "system", "consistency"], - &["ov", "system", "backend", "sync-status"], &["ov", "admin", "list-accounts"], ] { assert!( @@ -4708,9 +4656,6 @@ mod tests { &["ov", "system", "status"], &["ov", "system", "health"], &["ov", "system", "consistency"], - &["ov", "system", "backend"], - &["ov", "system", "backend", "sync-status"], - &["ov", "system", "backend", "sync-retry"], &["ov", "system", "crypto"], &["ov", "system", "crypto", "init-key"], &["ov", "session", "new"], @@ -5628,10 +5573,6 @@ mod tests { vec!["ov", "system", "consistency", "help"], "ov system consistency --help", ), - ( - vec!["ov", "system", "backend", "sync-status", "help"], - "ov system backend sync-status --help", - ), ] { let misuse = plain_help_misuse(&os_args(&args)) .unwrap_or_else(|| panic!("{args:?} should be plain-help misuse")); diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs index bab219ebc9..b09adc8c85 100644 --- a/crates/ragfs-python/src/lib.rs +++ b/crates/ragfs-python/src/lib.rs @@ -956,6 +956,7 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr { ragfs::core::Error::IsADirectory(_) => new_py_err("AGFSIsADirectoryError", msg), ragfs::core::Error::DirectoryNotEmpty(_) => new_py_err("AGFSDirectoryNotEmptyError", msg), ragfs::core::Error::InvalidOperation(_) => new_py_err("AGFSInvalidOperationError", msg), + ragfs::core::Error::NotSupported(_) => new_py_err("AGFSNotSupportedError", msg), ragfs::core::Error::Io(_) => new_py_err("AGFSIoError", msg), ragfs::core::Error::Plugin(_) => { // Check if the plugin error message contains known patterns @@ -1628,7 +1629,11 @@ impl RAGFSBindingClient { cache_config.stack_config(), )) .map_err(|error| { - PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}")) + if matches!(error, ragfs::core::Error::NotSupported(_)) { + to_py_err(error) + } else { + PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}")) + } })?; // Build the git service from inline config when present; otherwise fall @@ -1652,7 +1657,7 @@ impl RAGFSBindingClient { }) } - /// Stop mounted background services, then close the shared CacheRuntime. + /// Stop mounts requiring explicit cleanup, then close the shared CacheRuntime. fn close(&self, py: Python<'_>) -> PyResult<()> { let mountable = Arc::clone(&self.mountable); let cache_runtime = self.cache_runtime.clone(); @@ -2169,7 +2174,6 @@ impl RAGFSBindingClient { /// - "backups": nested dict matching BackendsConfig schema /// - "server_encryption_enabled": bool /// - "primary_encryption_enabled": bool - /// - "primary_redirects": list of redirect policy dicts #[pyo3(signature = (fstype, path, config=None))] fn mount( &self, @@ -2433,58 +2437,6 @@ impl RAGFSBindingClient { }) } - /// Query multi-write sync status under a file or directory path. - /// - /// Args: - /// path: Absolute AGFS path rooted at the mounted namespace - /// ctx: Optional FsContext dict (e.g. {"account_id": ...}) - /// - /// Returns: - /// A JSON-like dict describing pending and acknowledged backup sync state. - #[pyo3(signature = (path, ctx=None))] - fn system_sync_status( - &self, - py: Python<'_>, - path: String, - ctx: Option>, - ) -> PyResult> { - let fs_ctx = build_fs_context(ctx); - let mountable = self.mountable.clone(); - let result = self - .run_scoped(py, fs_ctx, move || async move { - mountable.system_sync_status(&path).await - }) - .map_err(to_py_err)?; - - Python::attach(|py| serde_json_to_py(py, &result)) - } - - /// Manually retry lagging multi-write backup sync operations under a path. - /// - /// Args: - /// path: Absolute AGFS path rooted at the mounted namespace - /// ctx: Optional FsContext dict (e.g. {"account_id": ...}) - /// - /// Returns: - /// A JSON-like dict summarizing retry results per target backend. - #[pyo3(signature = (path, ctx=None))] - fn system_sync_retry( - &self, - py: Python<'_>, - path: String, - ctx: Option>, - ) -> PyResult> { - let fs_ctx = build_fs_context(ctx); - let mountable = self.mountable.clone(); - let result = self - .run_scoped(py, fs_ctx, move || async move { - mountable.system_sync_retry(&path).await - }) - .map_err(to_py_err)?; - - Python::attach(|py| serde_json_to_py(py, &result)) - } - /// Calculate file digest (not yet implemented in ragfs). #[pyo3(signature = (path, algorithm="xxh3"))] fn digest(&self, path: String, algorithm: &str) -> PyResult> { diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml index 787c1ffcff..b941490a7c 100644 --- a/crates/ragfs/Cargo.toml +++ b/crates/ragfs/Cargo.toml @@ -18,11 +18,6 @@ path = "src/lib.rs" name = "ragfs-shell" path = "src/shell/main.rs" -[[test]] -name = "cache_wrapper" -path = "tests/cache_wrapper.rs" -required-features = ["cache", "test-utils"] - [[test]] name = "cache_runtime" path = "tests/cache_runtime.rs" @@ -67,6 +62,7 @@ filelocks = "0.1.0" # Data structures radix_trie = "0.2" +xxhash-rust = { version = "0.8", features = ["xxh3"] } # Error handling anyhow = "1.0" diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rs index 61c47051e0..4c5f4b67c6 100644 --- a/crates/ragfs/src/core/builder.rs +++ b/crates/ragfs/src/core/builder.rs @@ -200,6 +200,8 @@ pub async fn build_configured_stack( CacheNamespace::new(&cache.cachefs.namespace), cache.cachefs.policy, )) + } else if let Some(runtime) = runtime.as_ref() { + Arc::new(MountableFS::with_shared_cache_runtime(runtime.clone())) } else { Arc::new(MountableFS::new()) }; diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rs index 9ee84e5c22..43f10ac2c0 100644 --- a/crates/ragfs/src/core/errors.rs +++ b/crates/ragfs/src/core/errors.rs @@ -56,6 +56,10 @@ pub enum Error { #[error("invalid operation: {0}")] InvalidOperation(String), + /// Operation is intentionally unsupported by the current protocol. + #[error("not supported: {0}")] + NotSupported(String), + /// I/O error #[error("I/O error: {0}")] Io(#[from] io::Error), @@ -188,6 +192,7 @@ impl Error { Self::IsADirectory(_) => "is_a_directory", Self::DirectoryNotEmpty(_) => "directory_not_empty", Self::InvalidOperation(_) => "invalid_operation", + Self::NotSupported(_) => "not_supported", Self::Io(_) => "io", Self::Plugin(_) => "plugin", Self::Config(_) => "config", @@ -208,6 +213,11 @@ impl Error { pub fn invalid_operation(msg: impl Into) -> Self { Self::InvalidOperation(msg.into()) } + + /// Create a NotSupported error. + pub fn not_supported(msg: impl Into) -> Self { + Self::NotSupported(msg.into()) + } } #[cfg(test)] diff --git a/crates/ragfs/src/core/internal_names.rs b/crates/ragfs/src/core/internal_names.rs index dadfc36cde..f73ddd27d7 100644 --- a/crates/ragfs/src/core/internal_names.rs +++ b/crates/ragfs/src/core/internal_names.rs @@ -1,8 +1,7 @@ //! Static built-in internal names shared across the RAGFS stack. //! -//! Every module that references `.path.ovlock`, `.exact.ovlock.*`, `.redirect.json`, -//! or `.sync_log.json` must import these constants from this single source of truth. -//! There is no dynamic extension mechanism. +//! Every module that references `.path.ovlock` or `.exact.ovlock.*` must import +//! these constants from this single source of truth. /// Path-lock marker for directory-level locks. pub const PATH_LOCK_FILE: &str = ".path.ovlock"; @@ -10,21 +9,49 @@ pub const PATH_LOCK_FILE: &str = ".path.ovlock"; /// Prefix for exact (sidecar) lock files: `.exact.ovlock..`. pub const EXACT_LOCK_FILE_PREFIX: &str = ".exact.ovlock."; -/// Multi-write redirect metadata file. -pub const REDIRECT_FILE: &str = ".redirect.json"; - -/// Multi-write sync-log metadata file. -pub const SYNC_LOG_FILE: &str = ".sync_log.json"; - /// Returns `true` when `name` is a runtime path-lock file (`.path.ovlock` or `.exact.ovlock.*`). pub fn is_hidden_runtime_lock_name(name: &str) -> bool { // Reserve the whole prefix, including names not produced by the lock resolver. name == PATH_LOCK_FILE || name.starts_with(EXACT_LOCK_FILE_PREFIX) } -/// Returns `true` when `name` is any hidden internal name (lock files, redirect, sync-log). +/// Returns `true` when `name` is a hidden runtime lock name. pub fn is_hidden_internal_name(name: &str) -> bool { - is_hidden_runtime_lock_name(name) || name == REDIRECT_FILE || name == SYNC_LOG_FILE + is_hidden_runtime_lock_name(name) +} + +/// Returns `true` when a logical path belongs to V2 multi-write internals. +pub fn is_multiwrite_internal_path(path: &str) -> bool { + let Some(relative) = path.strip_prefix('/') else { + return false; + }; + let components = relative.split('/').collect::>(); + if components + .last() + .is_some_and(|name| is_hidden_internal_name(name)) + { + return true; + } + if matches!( + components.as_slice(), + ["local", "_system", ".multiwrite.json"] + ) { + return true; + } + if matches!( + components.as_slice(), + ["local", "_system", name] if name.starts_with(".multiwrite.catch-up.") + ) || matches!(components.as_slice(), ["local", "_system", "_system", ..]) + { + return true; + } + matches!( + components.as_slice(), + ["local", account, "_system", "partitions", ..] if !account.is_empty() + ) || matches!( + components.as_slice(), + ["local", account, "temp", ..] if !account.is_empty() + ) } #[cfg(test)] diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs index fb75b3d524..9e4a9be9d3 100644 --- a/crates/ragfs/src/core/mod.rs +++ b/crates/ragfs/src/core/mod.rs @@ -23,7 +23,7 @@ pub mod stats_wrapper; pub mod types; // Re-export commonly used types -pub use crate::multibackend::{FsContextResolver, MetaStateStore}; +pub use crate::multibackend::FsContextResolver; #[cfg(feature = "cache")] pub use builder::{ build_configured_stack, CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig, @@ -37,13 +37,12 @@ pub use encryption_wrapper::EncryptionWrappedFS; pub use errors::{Error, Result}; pub use filesystem::FileSystem; pub use mountable::MountableFS; -pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS, SyncMode}; +pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS}; pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin}; pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector}; pub use stats_wrapper::StatsWrappedFS; pub use types::{ - BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue, - EncryptionConfig, FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult, - ListSortBy, OperationItemConfig, PluginConfig, RedirectEntry, RedirectMeta, RedirectPolicy, - SortOrder, SyncLogEntry, SyncLogMeta, SyncOp, SyncType, TreeEntry, WriteFlag, + BackendItemConfig, BackendRole, BackendsConfig, ConfigParameter, ConfigValue, EncryptionConfig, + FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult, ListSortBy, PluginConfig, + SortOrder, TreeEntry, WriteFlag, }; diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs index 2b7f625ab2..32d9c201cd 100644 --- a/crates/ragfs/src/core/mountable.rs +++ b/crates/ragfs/src/core/mountable.rs @@ -6,7 +6,6 @@ use async_trait::async_trait; use radix_trie::{Trie, TrieCommon}; -use serde_json::Value; use std::collections::HashMap; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -14,10 +13,8 @@ use tokio::sync::RwLock; use tracing::warn; use crate::lock::{AutoPathLockAction, PathLockKind, PathLockManager, PathLockRequest}; -use crate::metrics::{ - lock_metrics, merge_metrics, operation_metrics, RagfsMetric, RagfsMetricValue, -}; -use crate::multibackend::factory::build_multi_write_fs; +use crate::metrics::{lock_metrics, merge_metrics, operation_metrics, RagfsMetric}; +use crate::multibackend::factory::build_inactive_multi_write_fs; use crate::multibackend::types::MultiBackendBuildContext; use crate::plugins::QueueFileSystem; use crate::shape::validate::ensure_backend_shape; @@ -88,6 +85,10 @@ pub struct MountableFS { /// Optional cache configuration shared by mounted filesystems. #[cfg(feature = "cache")] cache: Option, + + /// Optional shared runtime, independent of mount-level CacheFS enablement. + #[cfg(feature = "cache")] + cache_runtime: Option>, } #[cfg(feature = "cache")] @@ -169,9 +170,6 @@ impl MountableFS { if let Some(queuefs) = Self::as_queuefs_ref(mount_info.fs.as_ref()) { queuefs.shutdown().await?; } - if let Some(multiwrite) = Self::as_multiwrite(&mount_info.fs) { - multiwrite.shutdown().await?; - } Ok(()) } @@ -220,22 +218,6 @@ impl MountableFS { Self::encryption_handles_pathlock_ref(mount_info.fs.as_ref()) } - /// Query multi-write sync status for a mounted path. - pub async fn system_sync_status(&self, path: &str) -> Result { - let (mount_info, rel_path) = self.find_mount(path).await?; - let multiwrite = Self::as_multiwrite(&mount_info.fs) - .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?; - multiwrite.system_sync_status(&rel_path).await - } - - /// Manually retry pending multi-write sync operations under a mounted path. - pub async fn system_sync_retry(&self, path: &str) -> Result { - let (mount_info, rel_path) = self.find_mount(path).await?; - let multiwrite = Self::as_multiwrite(&mount_info.fs) - .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?; - multiwrite.system_sync_retry(&rel_path).await - } - /// Create a new MountableFS pub fn new() -> Self { Self { @@ -246,6 +228,22 @@ impl MountableFS { pathlock_manager: OnceLock::new(), #[cfg(feature = "cache")] cache: None, + #[cfg(feature = "cache")] + cache_runtime: None, + } + } + + /// Create a MountableFS that shares a runtime without enabling CacheFS. + #[cfg(feature = "cache")] + pub fn with_shared_cache_runtime(runtime: Arc) -> Self { + Self { + mounts: Arc::new(RwLock::new(Trie::new())), + registry: Arc::new(RwLock::new(HashMap::new())), + encryption_root_key: RwLock::new(None), + encryption_provider_type: RwLock::new(None), + pathlock_manager: OnceLock::new(), + cache: None, + cache_runtime: Some(runtime), } } @@ -263,10 +261,11 @@ impl MountableFS { encryption_provider_type: RwLock::new(None), pathlock_manager: OnceLock::new(), cache: Some(MountCacheConfig { - runtime, + runtime: runtime.clone(), namespace, policy, }), + cache_runtime: Some(runtime), } } @@ -448,9 +447,13 @@ impl MountableFS { stats: stats_collector, plugin_name: config.name.clone(), }; - + let mounted_fs = mount_info.fs.clone(); let mut mounts = self.mounts.write().await; mounts.insert(normalized_path, mount_info); + drop(mounts); + if let Some(multiwrite) = Self::as_multiwrite(&mounted_fs) { + multiwrite.activate(); + } Ok(()) } @@ -471,7 +474,7 @@ impl MountableFS { let pathlock_manager = self.pathlock_manager.get().cloned().ok_or_else(|| { Error::config("pathlock manager must be initialized before multi-write mount") })?; - build_multi_write_fs( + build_inactive_multi_write_fs( &self.registry, config, bc, @@ -480,6 +483,8 @@ impl MountableFS { enc_provider_type, pathlock_manager, backend_prefix: config.mount_path.clone(), + #[cfg(feature = "cache")] + cache_runtime: self.cache_runtime.clone(), }, ) .await @@ -617,28 +622,7 @@ impl MountableFS { metrics.extend(crate::metrics::cache_metrics(cache.metrics().snapshot())); } if let Some(multiwrite) = Self::as_multiwrite(&mount.fs) { - metrics.push(RagfsMetric { - name: "ragfs_multiwrite_background_tasks".into(), - labels: Default::default(), - value: RagfsMetricValue::Gauge(multiwrite.background_task_count() as f64), - }); - let routes = multiwrite.inner.read_route_metrics(); - for (route, key) in [ - ("primary", "primary_hits"), - ("backup", "backup_hits"), - ("redirect", "redirect_hits"), - ("miss", "misses"), - ] { - let count = routes[key].as_u64().ok_or_else(|| { - Error::internal(format!("invalid read-route counter '{key}'")) - })?; - metrics.push(RagfsMetric::counter( - "ragfs_multiwrite_read_routes_total", - &[("route", route)], - count, - 1.0, - )); - } + metrics.extend(multiwrite.metrics().await?); } } if let Some(manager) = self.pathlock_manager.get() { @@ -1250,9 +1234,7 @@ impl FileSystem for MountableFS { mod tests { use super::*; use crate::core::BackendItemConfig; - #[cfg(feature = "cache")] use crate::core::ConfigValue; - use crate::core::RedirectPolicy; use crate::shape::SHAPE_MANIFEST_PATH; use serde_json::Value; use std::collections::HashMap; @@ -1274,29 +1256,66 @@ mod tests { mount_path: mount_path.to_string(), params: HashMap::new(), backups: Some(BackendsConfig { - sync_type: "async".to_string(), - write_ack_count: None, - write_ack_timeout_ms: None, - write_concurrency: None, - retry_interval_ms: None, - retry_backoff_base_ms: None, - retry_max_retries_per_round: None, - retry_quarantine_after_failures: None, - read_probe_cache_ttl_ms: None, + namespace: "default".to_string(), + initial_partitions: 16, + checkpoint_interval_secs: 86_400, + provider: "filesystem".to_string(), items: vec![BackendItemConfig { name: "backup1".to_string(), backend: backup_backend.to_string(), params: Value::Null, - timeout: None, encryption: None, - operations: None, - excludes: None, }], }), ..PluginConfig::default() } } + /// Build a two-backup LocalFS configuration from three persistent directories. + fn local_multiwrite_test_config( + primary_dir: &std::path::Path, + backup_dirs: [(&str, &std::path::Path); 2], + ) -> PluginConfig { + let mut config = multiwrite_test_config("localfs", "localfs", "/local"); + config.params.insert( + "local_dir".to_string(), + ConfigValue::String(primary_dir.to_string_lossy().into_owned()), + ); + config.backups.as_mut().unwrap().initial_partitions = 1; + config.backups.as_mut().unwrap().items = backup_dirs + .into_iter() + .map(|(name, path)| BackendItemConfig { + name: name.to_string(), + backend: "localfs".to_string(), + params: serde_json::json!({ + "local_dir": path.to_string_lossy(), + }), + encryption: None, + }) + .collect(); + config + } + + /// Mount one LocalFS multi-write configuration with the shared lock provider. + async fn mount_local_multiwrite_for_test( + config: PluginConfig, + provider: Arc, + ) -> Arc { + use crate::lock::{PathLockConfig, PathLockManager}; + use crate::plugins::{memfs::MemFileSystem, LocalFSPlugin}; + + let mfs = Arc::new(MountableFS::new()); + mfs.register_plugin(LocalFSPlugin::new()).await; + let manager = Arc::new(PathLockManager::new( + Arc::new(MemFileSystem::new()), + provider, + PathLockConfig::default(), + )); + mfs.set_pathlock_manager(manager).await; + mfs.mount(config).await.unwrap(); + mfs + } + // Mock filesystem for testing struct MockFS { name: String, @@ -1338,10 +1357,10 @@ mod tests { } async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result> { - if path == SHAPE_MANIFEST_PATH - || path.ends_with(".redirect.json") - || path.ends_with(".sync_log.json") - { + if path == "/_system/accounts.json" { + return Ok(br#"{"accounts":{}}"#.to_vec()); + } + if path == SHAPE_MANIFEST_PATH || path == "/_system/.multiwrite.json" { return Err(Error::not_found(path)); } Ok(self.name.as_bytes().to_vec()) @@ -1373,6 +1392,9 @@ mod tests { } async fn stat(&self, path: &str) -> Result { + if path == "/_system/.multiwrite.json" { + return Err(Error::not_found(path)); + } Ok(FileInfo::new_file(path.to_string(), 0, 0o644)) } @@ -1561,33 +1583,6 @@ mod tests { mfs } - /// Create a MountableFS backed by the real in-memory plugin. - async fn mounted_memfs(mount_path: &str) -> MountableFS { - use crate::plugins::MemFSPlugin; - - let mfs = MountableFS::new(); - mfs.register_plugin(MemFSPlugin).await; - mfs.mount(test_config("memfs", mount_path)).await.unwrap(); - mfs - } - - /// Create a cache-enabled MountableFS backed by the real in-memory plugin. - #[cfg(feature = "cache")] - async fn mounted_cached_memfs(namespace: &str, mount_path: &str) -> MountableFS { - use crate::cache::{CacheNamespace, CachePolicy}; - use crate::cache_runtime::CacheRuntime; - use crate::plugins::MemFSPlugin; - - let mfs = MountableFS::with_cache_runtime( - CacheRuntime::memory(), - CacheNamespace::new(namespace), - CachePolicy::default(), - ); - mfs.register_plugin(MemFSPlugin).await; - mfs.mount(test_config("memfs", mount_path)).await.unwrap(); - mfs - } - /// Create a MountableFS with two mounted mock plugins. async fn mounted_two_mocks() -> MountableFS { let mfs = MountableFS::new(); @@ -1733,6 +1728,32 @@ mod tests { ); } + /// Verifies service shutdown leaves multi-write workers to process teardown. + #[tokio::test] + async fn shutdown_does_not_stop_multiwrite_runtime() { + use crate::lock::MemoryPathLockProvider; + + let primary = tempfile::tempdir().unwrap(); + let backup_a = tempfile::tempdir().unwrap(); + let backup_b = tempfile::tempdir().unwrap(); + let config = local_multiwrite_test_config( + primary.path(), + [("backup-a", backup_a.path()), ("backup-b", backup_b.path())], + ); + let mfs = + mount_local_multiwrite_for_test(config, Arc::new(MemoryPathLockProvider::new())).await; + let mount = mfs.mounts.read().await.get("/local").unwrap().clone(); + + mfs.shutdown().await.unwrap(); + + assert_eq!( + MountableFS::as_multiwrite(&mount.fs) + .unwrap() + .background_task_count(), + 1 + ); + } + #[cfg(feature = "cache")] #[tokio::test] async fn shutdown_reaches_cache_queuefs_through_stats_wrapper() { @@ -1949,115 +1970,6 @@ mod tests { ); } - #[cfg(feature = "cache")] - #[tokio::test] - async fn encrypted_multiwrite_mount_does_not_install_plaintext_cache() { - use crate::cache::{CacheNamespace, CachePolicy}; - use crate::cache_runtime::CacheRuntime; - use crate::lock::{ - MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, - }; - use crate::plugins::MemFSPlugin; - - let mfs = Arc::new(MountableFS::with_cache_runtime( - CacheRuntime::memory(), - CacheNamespace::new("encrypted-multiwrite-test"), - CachePolicy::default(), - )); - mfs.register_plugin(MemFSPlugin).await; - mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await; - let provider: Arc = Arc::new(MemoryPathLockProvider::new()); - let manager = Arc::new(PathLockManager::new( - mfs.clone() as Arc, - provider, - PathLockConfig::default(), - )); - mfs.set_pathlock_manager(manager).await; - - let mut config = multiwrite_test_config("memfs", "memfs", "/local"); - config.server_encryption_enabled = true; - config.primary_encryption_enabled = true; - mfs.mount(config).await.unwrap(); - - let mounts = mfs.mounts.read().await; - let mount_info = mounts.get("/local").expect("mounted entry should exist"); - let stats = (mount_info.fs.as_ref() as &dyn std::any::Any) - .downcast_ref::() - .expect("stats wrapper should be outermost"); - assert!( - (stats.inner_fs().as_ref() as &dyn std::any::Any) - .downcast_ref::() - .is_some(), - "encrypted multi-write must not install a plaintext cache outside MultiWriteWrappedFS" - ); - } - - #[cfg(feature = "cache")] - #[tokio::test] - async fn cached_unencrypted_multiwrite_keeps_admin_and_copy_fast_paths() { - use crate::cache::{CacheNamespace, CachePolicy}; - use crate::cache_runtime::CacheRuntime; - use crate::core::{FsContextInner, FS_CTX}; - use crate::plugins::MemFSPlugin; - - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache_runtime( - CacheRuntime::memory(), - CacheNamespace::new("cached-multiwrite-test"), - CachePolicy::default(), - ))) - .await; - mfs.register_plugin(MemFSPlugin).await; - - let mut config = multiwrite_test_config("memfs", "memfs", "/local"); - config.backups.as_mut().unwrap().items[0].name = "backup1".to_string(); - mfs.mount(config).await.unwrap(); - - let ctx = Arc::new(FsContextInner::new("acct")); - FS_CTX - .scope(ctx.clone(), async { - let status = mfs.system_sync_status("/local").await.unwrap(); - assert_eq!(status["path"], "/"); - assert_eq!(status["entry_count"], 0); - - let retry = mfs.system_sync_retry("/local").await.unwrap(); - assert_eq!(retry["path"], "/"); - assert_eq!(retry["retried"], 0); - - mfs.mkdir("/local/docs", 0o755).await.unwrap(); - mfs.write("/local/docs/src.md", b"copied", 0, WriteFlag::Create) - .await - .unwrap(); - - assert!(mfs - .copy_within_mount("/local/docs/src.md", "/local/docs/dst.md") - .await - .unwrap()); - assert_eq!( - mfs.read("/local/docs/dst.md", 0, 0).await.unwrap(), - b"copied" - ); - }) - .await; - - mfs.unmount("/local").await.unwrap(); - assert!(mfs.list_mounts().await.is_empty()); - } - - #[tokio::test] - async fn multiwrite_mount_without_pathlock_returns_config_error() { - use crate::plugins::MemFSPlugin; - - let mfs = MountableFS::new(); - mfs.register_plugin(MemFSPlugin).await; - - let error = mfs - .mount(multiwrite_test_config("memfs", "memfs", "/local")) - .await - .unwrap_err(); - - assert!(matches!(error, Error::Config(_))); - } - #[tokio::test] async fn test_mount_duplicate_error() { let mfs = Arc::new(MountableFS::new()); @@ -2346,180 +2258,6 @@ mod tests { assert_eq!(result[0].rel_path, "a/b/c.txt"); } - #[tokio::test] - async fn test_build_multi_write_fs_rejects_primary_encryption_disable() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await; - - let backups = BackendsConfig { - sync_type: "async".to_string(), - write_ack_count: None, - write_ack_timeout_ms: None, - write_concurrency: None, - retry_interval_ms: None, - retry_backoff_base_ms: None, - retry_max_retries_per_round: None, - retry_quarantine_after_failures: None, - read_probe_cache_ttl_ms: None, - items: vec![BackendItemConfig { - name: "backup1".to_string(), - backend: "backupfs".to_string(), - params: serde_json::Value::Null, - timeout: None, - encryption: None, - operations: None, - excludes: None, - }], - }; - let config = PluginConfig { - name: "primary".to_string(), - mount_path: "/local".to_string(), - params: HashMap::new(), - backups: Some(backups.clone()), - server_encryption_enabled: true, - ..PluginConfig::default() - }; - - let result = mfs.build_multi_write_fs(&config, &backups).await; - assert!(result.is_err()); - assert!(matches!(result.err(), Some(Error::Config(_)))); - } - - #[tokio::test] - async fn encrypted_multiwrite_reports_pathlock_ownership() { - use crate::core::{FsContextInner, PathLockContext, FS_CTX}; - use crate::lock::{ - MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, - }; - use crate::plugins::MemFSPlugin; - - let mfs = Arc::new(MountableFS::new()); - mfs.register_plugin(MemFSPlugin).await; - mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await; - let provider: Arc = Arc::new(MemoryPathLockProvider::new()); - let manager = Arc::new(PathLockManager::new( - mfs.clone() as Arc, - provider, - PathLockConfig::default(), - )); - mfs.set_pathlock_manager(manager.clone()).await; - - let mut config = multiwrite_test_config("memfs", "memfs", "/local"); - config.server_encryption_enabled = true; - config.primary_encryption_enabled = true; - mfs.mount(config).await.unwrap(); - - assert!(mfs.encryption_handles_pathlock("/local/file.txt").await); - - let outer_ctx = Arc::new(FsContextInner::new("tenant")); - let outer = FS_CTX - .scope(outer_ctx, async { - manager - .acquire_exact("/local/tenant/file.txt", std::time::Duration::ZERO, None) - .await - }) - .await - .unwrap(); - let ctx = Arc::new(FsContextInner::with_pathlock( - "tenant", - PathLockContext { - lease_ref: Some(outer.lease.lease_ref.clone()), - disable_auto_pathlock: false, - }, - )); - FS_CTX - .scope(ctx, async { - mfs.write("/local/tenant/file.txt", b"content", 0, WriteFlag::Create) - .await - }) - .await - .unwrap(); - manager.release(&outer).await.unwrap(); - } - - #[tokio::test] - async fn test_build_multi_write_fs_rejects_exclude_target_field() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - - let backups = BackendsConfig { - sync_type: "async".to_string(), - write_ack_count: None, - write_ack_timeout_ms: None, - write_concurrency: None, - retry_interval_ms: None, - retry_backoff_base_ms: None, - retry_max_retries_per_round: None, - retry_quarantine_after_failures: None, - read_probe_cache_ttl_ms: None, - items: vec![BackendItemConfig { - name: "backup1".to_string(), - backend: "backupfs".to_string(), - params: serde_json::Value::Null, - timeout: None, - encryption: None, - operations: None, - excludes: Some(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.tmp$".to_string()], - target: Some(vec!["should-not-exist".to_string()]), - }]), - }], - }; - let config = PluginConfig { - name: "primary".to_string(), - mount_path: "/local".to_string(), - params: HashMap::new(), - backups: Some(backups.clone()), - ..PluginConfig::default() - }; - - let result = mfs.build_multi_write_fs(&config, &backups).await; - assert!(result.is_err()); - assert!(matches!(result.err(), Some(Error::Config(_)))); - } - - #[tokio::test] - async fn test_build_multi_write_fs_rejects_reserved_primary_backup_name() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - - let backups = BackendsConfig { - sync_type: "async".to_string(), - write_ack_count: None, - write_ack_timeout_ms: None, - write_concurrency: None, - retry_interval_ms: None, - retry_backoff_base_ms: None, - retry_max_retries_per_round: None, - retry_quarantine_after_failures: None, - read_probe_cache_ttl_ms: None, - items: vec![BackendItemConfig { - name: "primary".to_string(), - backend: "backupfs".to_string(), - params: serde_json::Value::Null, - timeout: None, - encryption: None, - operations: None, - excludes: None, - }], - }; - let config = PluginConfig { - name: "primary".to_string(), - mount_path: "/local".to_string(), - params: HashMap::new(), - backups: Some(backups.clone()), - ..PluginConfig::default() - }; - - let result = mfs.build_multi_write_fs(&config, &backups).await; - assert!(result.is_err()); - assert!(matches!(result.err(), Some(Error::Config(_)))); - } - #[tokio::test] async fn test_single_backend_mount_wraps_stats_outside_encryption() { use crate::lock::{ @@ -2552,83 +2290,4 @@ mod tests { "single-backend encrypted mount should place encryption under stats" ); } - - #[tokio::test] - async fn test_multiwrite_mount_wraps_stats_outside_multiwrite() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - - mfs.mount(multiwrite_test_config("primary", "backupfs", "/local")) - .await - .unwrap(); - - let mounts = mfs.mounts.read().await; - let mount_info = mounts.get("/local").expect("mounted entry should exist"); - let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any) - .downcast_ref::() - .expect("stats wrapper should be outermost"); - assert!( - (wrapped.inner_fs().as_ref() as &dyn std::any::Any) - .downcast_ref::() - .is_some(), - "multi-write mount should place MultiWriteWrappedFS directly under stats" - ); - } - - #[tokio::test] - async fn test_multiwrite_raw_access_is_rejected() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - - mfs.mount(multiwrite_test_config("primary", "backupfs", "/local")) - .await - .unwrap(); - - let read_err = mfs.read_raw("/local/data.txt", 0, 0).await.unwrap_err(); - assert!(matches!(read_err, Error::InvalidOperation(_))); - - let write_err = mfs - .write_raw("/local/data.txt", b"raw", WriteFlag::Create) - .await - .unwrap_err(); - assert!(matches!(write_err, Error::InvalidOperation(_))); - } - - #[tokio::test] - async fn test_mountable_multiwrite_admin_smoke() { - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await; - mfs.register_plugin(MockPlugin::new("primary")).await; - mfs.register_plugin(MockPlugin::new("backupfs")).await; - - mfs.mount(multiwrite_test_config("primary", "backupfs", "/local")) - .await - .unwrap(); - - let status = crate::core::FS_CTX - .scope( - Arc::new(crate::core::context::FsContextInner::new( - "acct".to_string(), - )), - async { mfs.system_sync_status("/local").await.unwrap() }, - ) - .await; - assert_eq!(status["path"], "/"); - assert_eq!(status["entry_count"], 0); - assert_eq!(status["pending_target_count"], 0); - assert_eq!(status["capabilities"]["multi_instance_safe"], false); - - let retry = crate::core::FS_CTX - .scope( - Arc::new(crate::core::context::FsContextInner::new( - "acct".to_string(), - )), - async { mfs.system_sync_retry("/local").await.unwrap() }, - ) - .await; - assert_eq!(retry["path"], "/"); - assert_eq!(retry["retried"], 0); - assert_eq!(retry["failed"], 0); - } } diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs index c8f8149d8b..8d5553a378 100644 --- a/crates/ragfs/src/core/multibackend_wrapper.rs +++ b/crates/ragfs/src/core/multibackend_wrapper.rs @@ -1,1415 +1,591 @@ -//! Multi-write wrapper — routes operations across primary and backup backends. -//! -//! Implements `MultiWriteWrappedFS` which handles: -//! - Write fanout to primary + backup backends (sync/async) -//! - Read routing with priority-based fallback chain -//! - Redirect policy evaluation -//! - Exclude policy filtering -//! - `.redirect.json` / `.sync_log.json` metadata management +//! Primary filesystem wrapper that submits V2 multi-write metadata events. -use std::collections::{HashMap, HashSet}; -use std::future::Future; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; -use futures::stream::{self, StreamExt}; -use regex::Regex; -use serde_json::Value; -use tokio::sync::{Mutex, Notify}; -use tracing::warn; +use uuid::Uuid; use super::context::{FsContext, FS_CTX}; use super::encryption_wrapper::EncryptionWrappedFS; use super::errors::{Error, Result}; -use super::filesystem::{ - apply_read_dir_options, normalize_prefix_path, paginate_entries, relative_depth, - relative_match_file, FileSystem, -}; +use super::filesystem::{normalize_prefix_path, FileSystem}; use super::types::{ - BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepOptions, GrepResult, - ListSortBy, OperationItemConfig, RedirectEntry, RedirectPolicy, SortOrder, SyncLogEntry, - SyncOp, SyncType, TreeEntry, WriteFlag, -}; -use crate::core::glob::{ - compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob, + BackendRole, FileInfo, GlobPage, GrepOptions, GrepResult, ListSortBy, SortOrder, TreeEntry, + WriteFlag, }; -use crate::core::internal_names::is_hidden_internal_name; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::metrics::{RagfsMetric, RagfsMetricValue}; +use crate::multibackend::catch_up::CatchUpTarget; use crate::multibackend::meta::{ - current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver, - MetaStateStore, PathSerializer, MULTIWRITE_INTERNAL_NAMES, + current_required_ctx, DefaultFsContextResolver, FsContextResolver, MetadataStore, + MultiWriteWorker, }; +use crate::multibackend::migration::{FullDataImporter, ProtocolDetector}; +use crate::multibackend::model::{ + PartitionState, PartitionsManifest, PendingEvent, PendingEventKind, ProtocolState, + ProtocolStatus, ProtocolVersion, SegmentEventType, SegmentManifest, +}; +use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider}; +use crate::multibackend::runtime::MultiWriteRuntime; -mod retry; -mod routing; -#[cfg(test)] -mod tests; - -/// Default chunk size used when copying file state from primary to backup. -const DEFAULT_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024; -/// Default retry loop interval. -const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(30); -/// Default retry backoff base in milliseconds. -const DEFAULT_RETRY_BACKOFF_BASE_MS: u64 = 1000; -/// Default maximum retries per file per retry round. -const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3; -/// Default failure threshold before a target is quarantined. -const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9; -/// Default wait timeout when shutting down background tasks. -const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5); -/// Maximum concurrent metadata reads when merging redirect entries into tree output. -const DEFAULT_TREE_REDIRECT_META_CONCURRENCY: usize = 32; - -#[derive(Debug)] -struct SyncFanoutTaskResult { - backend: String, - result: Result<()>, -} - -#[derive(Clone)] -enum BackupWriteOp { - Replay(SyncOp), - WriteFile { - data: Arc>, - offset: u64, - flags: WriteFlag, - }, - EnsureParentDirs { - mode: u32, - }, -} - -impl BackupWriteOp { - /// Apply this backup-side operation under the provided request context. - async fn apply( - &self, - inner: &Inner, - backup: Arc, - path: &str, - ctx: &FsContext, - ) -> Result<()> { - match self { - Self::Replay(op) => { - op.replay(inner.primary().backend.clone(), backup, path, ctx) - .await - } - Self::WriteFile { - data, - offset, - flags, - } => { - let data = Arc::clone(data); - let offset = *offset; - let flags = *flags; - FS_CTX - .scope(ctx.clone(), async move { - backup.ensure_parent_dirs(path, 0o755).await?; - backup.write(path, data.as_slice(), offset, flags).await?; - Ok(()) - }) - .await - } - Self::EnsureParentDirs { mode } => { - let mode = *mode; - FS_CTX - .scope(ctx.clone(), async move { - backup.ensure_parent_dirs(path, mode).await - }) - .await - } - } - } -} - -/// Cloned target data passed to fanout strategies without borrowing `Inner`. -#[derive(Clone)] -struct FanoutTarget { - name: String, - backend: Arc, -} - -/// Effective sync work item with resolved target backend names. -pub(crate) struct SyncWorkEntry { - pub(crate) file_path: String, - pub(crate) entry: SyncLogEntry, - pub(crate) targets: Vec, -} - -#[derive(Clone, Copy)] -enum ReadRouteSource { - Backup, - Primary, - Redirect, - Miss, -} - -/// Builder-style sync mode configuration. -pub enum SyncMode { - /// Synchronous fanout requiring backup acknowledgement. - Sync { - /// Minimum backup acknowledgements required for a successful write. - ack_count: usize, - /// Maximum time to wait for backup acknowledgements, in milliseconds. - timeout_ms: u64, - }, - /// Asynchronous fanout with background retry. - Async, -} +const COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024; -/// A backend entry within the multi-write wrapper. +/// One configured primary or backup filesystem. pub struct BackendEntry { - /// Logical name (globally unique) + /// Logical backend name. pub name: String, - /// Role: Primary or Backup + /// Backend role. pub role: BackendRole, - /// The backend filesystem handle (may be encrypted) + /// Filesystem handle, including encryption when configured. pub backend: Arc, - /// Optional raw backend handle used by primary verbatim copy fast-paths. + /// Optional raw filesystem handle. pub raw_backend: Option>, - /// Operations this backend participates in (only for Backup) - pub operations: Vec, - /// Exclude policies (only for Backup) - pub excludes: Vec, -} - -impl BackendEntry { - /// Check if this backend participates in read operations. - fn participates_in_read(&self) -> bool { - self.operations.iter().any(|op| op.operation == "read") - } - - /// Check if this backend participates in write operations. - /// Backups default to write-enabled when operations is empty. - fn participates_in_write(&self) -> bool { - if self.operations.is_empty() { - true - } else { - self.operations.iter().any(|op| op.operation == "write") - } - } - - /// Get read priority (lower = higher priority). Returns None if not read-enabled. - fn read_priority(&self) -> Option { - self.operations - .iter() - .find(|op| op.operation == "read") - .map(|op| op.priority) - } - - /// Convert this backend entry into a fanout target. - fn fanout_target(&self) -> FanoutTarget { - FanoutTarget { - name: self.name.clone(), - backend: self.backend.clone(), - } - } -} - -/// File policy trait — shared by redirects and excludes. -pub trait FilePolicy { - /// Check if this policy matches the given file. - fn matches(&self, path: &str, size: u64) -> bool; -} - -impl FilePolicy for RedirectPolicy { - fn matches(&self, path: &str, size: u64) -> bool { - match self { - RedirectPolicy::FileOverSizePolicy { max_size_mb, .. } => { - let max_bytes = max_size_mb * 1024 * 1024; - size > max_bytes - } - RedirectPolicy::FileExtensionPolicy { extensions, .. } => { - let name = file_name(path); - extensions.iter().any(|ext_pattern| { - if let Ok(re) = Regex::new(ext_pattern) { - re.is_match(name) - } else { - name.ends_with(ext_pattern.as_str()) - } - }) - } - } - } -} - -impl SyncOp { - /// Replay this operation on one backup backend using the original request semantics. - async fn replay( - &self, - primary: Arc, - backup: Arc, - file_path: &str, - ctx: &FsContext, - ) -> Result<()> { - match self { - SyncOp::SyncFile { size } => { - let size = *size; - copy_current_primary_state(primary, backup, file_path, size, ctx).await - } - SyncOp::Create => { - FS_CTX - .scope(ctx.clone(), async { backup.create(file_path).await }) - .await - } - SyncOp::Mkdir { mode } => { - let mode = *mode; - FS_CTX - .scope(ctx.clone(), async { backup.mkdir(file_path, mode).await }) - .await - } - SyncOp::Remove => { - match FS_CTX - .scope(ctx.clone(), async { backup.remove(file_path).await }) - .await - { - Ok(()) | Err(Error::NotFound(_)) => Ok(()), - Err(e) => Err(e), - } - } - SyncOp::RemoveAll => { - match FS_CTX - .scope(ctx.clone(), async { backup.remove_all(file_path).await }) - .await - { - Ok(()) | Err(Error::NotFound(_)) => Ok(()), - Err(e) => Err(e), - } - } - SyncOp::Rename { to } => { - let to = to.clone(); - FS_CTX - .scope(ctx.clone(), async { - backup.ensure_parent_dirs(&to, 0o755).await?; - backup.rename(file_path, &to).await - }) - .await - } - SyncOp::Chmod { mode } => { - let mode = *mode; - FS_CTX - .scope(ctx.clone(), async { backup.chmod(file_path, mode).await }) - .await - } - } - } -} - -/// Copy one file between two filesystem handles in bounded-size chunks. -async fn copy_file_state( - source: Arc, - source_path: &str, - destination: Arc, - destination_path: &str, - size: u64, - ctx: &FsContext, -) -> Result<()> { - FS_CTX - .scope(ctx.clone(), async { - destination - .ensure_parent_dirs(destination_path, 0o755) - .await?; - if size == 0 { - if destination.exists(destination_path).await { - return destination.truncate(destination_path, 0).await; - } - return destination.create(destination_path).await; - } - - let mut offset = 0u64; - while offset < size { - let chunk_len = (size - offset).min(DEFAULT_COPY_CHUNK_SIZE as u64); - let chunk = source.read(source_path, offset, chunk_len).await?; - if chunk.is_empty() { - if offset == 0 { - if destination.exists(destination_path).await { - destination.truncate(destination_path, 0).await?; - } else { - destination.create(destination_path).await?; - } - } - break; - } - let flag = if offset == 0 { - WriteFlag::Create - } else { - WriteFlag::None - }; - destination - .write(destination_path, &chunk, offset, flag) - .await?; - offset = offset.saturating_add(chunk.len() as u64); - } - Ok(()) - }) - .await -} - -/// Copy the current file state from primary to backup in bounded-size chunks. -async fn copy_current_primary_state( - primary: Arc, - backup: Arc, - file_path: &str, - size: u64, - ctx: &FsContext, -) -> Result<()> { - copy_file_state(primary, file_path, backup, file_path, size, ctx).await -} - -/// Copy one file within the primary raw backend without decrypting and re-encrypting bytes. -async fn copy_raw_primary_state( - primary_raw: Arc, - source_path: &str, - destination_path: &str, - ctx: &FsContext, -) -> Result<()> { - let raw_size = FS_CTX - .scope(ctx.clone(), async { - let source_info = primary_raw.stat(source_path).await?; - if source_info.is_dir { - return Err(Error::IsADirectory(source_path.to_string())); - } - Ok::(source_info.size) - }) - .await?; - - copy_file_state( - primary_raw.clone(), - source_path, - primary_raw, - destination_path, - raw_size, - ctx, - ) - .await } -/// Inner state shared via Arc for async spawn and retry_loop. pub(crate) struct Inner { - /// All backend entries (primary at index 0) - backends: Vec, - /// Index of the primary backend - primary_idx: usize, - /// Sync type: Async or Sync. - sync_type: SyncType, - /// Minimum backup ack count for sync mode - write_ack_count: usize, - /// Timeout for waiting backup ack in sync mode (ms) - write_ack_timeout_ms: u64, - /// Semaphore for async write concurrency control - write_sem: Option>, - /// Primary redirect policies - redirects: Vec, - /// Metadata store (encrypted via primary_backend) - pub(crate) meta_store: MetaStateStore, - /// Per-path serialization queues - path_queues: PathSerializer, - /// Directories that currently have outstanding retry work. - pending_dirs: Mutex>, - /// Retry loop interval. - retry_interval: Duration, - /// Base retry backoff in milliseconds. - pub(crate) retry_backoff_base_ms: u64, - /// Maximum retry attempts for one target in one round. - pub(crate) max_retry_per_round: usize, - /// Failure threshold before quarantining one target. - quarantine_after_failures: u32, - /// Number of background tasks currently in flight. - background_tasks: AtomicUsize, - /// Notifier fired when background task count reaches zero. - idle_notify: Notify, - /// Read route hit metrics. - read_backup_hits: AtomicU64, - read_primary_hits: AtomicU64, - read_redirect_hits: AtomicU64, - read_misses: AtomicU64, - /// Cancellation flag for the background retry loop. - retry_cancelled: AtomicBool, - /// Wake-up signal used to stop retry_loop promptly on drop. - retry_shutdown: Notify, + primary: BackendEntry, + runtime: MultiWriteRuntime, + ctx_resolver: Arc, } -/// Multi-write wrapped filesystem. +/// Filesystem wrapper that commits primary operations before queuing V2 events. +#[derive(Clone)] pub struct MultiWriteWrappedFS { pub(crate) inner: Arc, + metadata_store: Arc, + metadata_provider: Arc, } -/// Builder for `MultiWriteWrappedFS`. +/// Builder for one V2 multi-write wrapper and its flush runtime. pub struct MultiWriteWrappedFSBuilder { primary_backend: Arc, primary_raw_backend: Option>, backup_entries: Vec, - redirects: Vec, - sync_mode: SyncMode, - write_concurrency: Option, - retry_interval: Duration, - retry_backoff_base_ms: u64, - max_retry_per_round: usize, - quarantine_after_failures: u32, + metadata_store: Option>, + metadata_provider: Option>, + initial_partitions: u32, + checkpoint_interval: Duration, ctx_resolver: Arc, } impl MultiWriteWrappedFSBuilder { - /// Attach the raw primary backend so the wrapper can do verbatim primary copies. + /// Attach the raw primary backend used by internal operations and physical copies. pub fn with_primary_raw_backend(mut self, primary_raw_backend: Arc) -> Self { self.primary_raw_backend = Some(primary_raw_backend); self } - /// Set backup backend entries on the builder. + /// Attach configured backup handles for later catch-up workers. pub fn with_backups(mut self, backup_entries: Vec) -> Self { self.backup_entries = backup_entries; self } - /// Set redirect policies for the primary backend. - pub fn with_redirects(mut self, redirects: Vec) -> Self { - self.redirects = redirects; + /// Attach the metadata store used for account initialization and routing. + pub fn with_metadata_store(mut self, metadata_store: Arc) -> Self { + self.metadata_store = Some(metadata_store); self } - /// Select the sync mode used by write fanout. - pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self { - self.sync_mode = sync_mode; + /// Attach the provider used by the background flush worker. + pub fn with_metadata_provider( + mut self, + metadata_provider: Arc, + ) -> Self { + self.metadata_provider = Some(metadata_provider); self } - /// Set the maximum number of concurrent async backup writes. - pub fn write_concurrency(mut self, write_concurrency: Option) -> Self { - self.write_concurrency = write_concurrency; + /// Set the initial metadata partition count for newly observed accounts. + pub fn initial_partitions(mut self, initial_partitions: u32) -> Self { + self.initial_partitions = initial_partitions; self } - /// Configure retry loop interval. - pub fn retry_interval(mut self, retry_interval: Duration) -> Self { - self.retry_interval = retry_interval; + /// Set the interval between checkpoint discovery rounds. + pub fn checkpoint_interval(mut self, checkpoint_interval: Duration) -> Self { + self.checkpoint_interval = checkpoint_interval; self } - /// Configure retry backoff base duration in milliseconds. - pub fn retry_backoff_base_ms(mut self, retry_backoff_base_ms: u64) -> Self { - self.retry_backoff_base_ms = retry_backoff_base_ms; - self - } - - /// Configure the maximum number of retries per round. - pub fn max_retry_per_round(mut self, max_retry_per_round: usize) -> Self { - self.max_retry_per_round = max_retry_per_round.max(1); - self - } - - /// Configure quarantine threshold for one path/backup pair. - pub fn quarantine_after_failures(mut self, quarantine_after_failures: u32) -> Self { - self.quarantine_after_failures = quarantine_after_failures.max(1); - self - } - - /// Configure the context resolver used by retry and admin background paths. + /// Configure the resolver for mount-relative foreground paths. pub fn ctx_resolver(mut self, ctx_resolver: Arc) -> Self { self.ctx_resolver = ctx_resolver; self } - /// Build the multi-write wrapper and start the retry loop when needed. - pub fn build(self) -> Result { - let mut backends = Vec::new(); - backends.push(BackendEntry { + /// Build the wrapper after asynchronously starting its V2 runtime. + pub async fn build(self) -> Result { + let fs = self.build_inactive().await?; + fs.activate(); + Ok(fs) + } + + /// Build the wrapper while keeping its V2 runtime inactive. + pub(crate) async fn build_inactive(self) -> Result { + let metadata_store = self + .metadata_store + .ok_or_else(|| Error::config("multi-write metadata store is required"))?; + let metadata_provider = self + .metadata_provider + .ok_or_else(|| Error::config("multi-write metadata provider is required"))?; + let backup_names: Vec = self + .backup_entries + .iter() + .map(|entry| entry.name.clone()) + .collect(); + let catch_up_targets = self + .backup_entries + .iter() + .map(|entry| CatchUpTarget { + backend_id: entry.name.clone(), + backend: entry.backend.clone(), + }) + .collect(); + let protocol = ProtocolDetector::new(metadata_store.clone()) + .detect() + .await?; + let configured_backups = backup_names.iter().collect::>(); + for account in metadata_store.initialized_accounts().await? { + let path = metadata_store.paths().account_manifest(&account)?.0; + let manifest: PartitionsManifest = match metadata_store.read_json(&path).await { + Ok(manifest) => manifest, + Err(Error::NotFound(_)) => continue, + Err(error) => return Err(error), + }; + for &partition_id in manifest.partitions.keys() { + let path = metadata_store + .paths() + .segment_manifest(&account, partition_id)? + .0; + let segments: SegmentManifest = metadata_store.read_json(&path).await?; + segments.validate()?; + let persisted = segments.backend_states.keys().collect::>(); + if persisted != configured_backups { + return Err(Error::not_supported(format!( + "backup membership changed for account {account}" + ))); + } + } + bootstrap_account( + metadata_provider.as_ref(), + metadata_store.as_ref(), + &account, + &manifest, + ) + .await?; + } + let runtime = MultiWriteRuntime::prepare( + metadata_store.clone(), + metadata_provider.clone(), + self.initial_partitions, + backup_names.clone(), + Some(self.primary_backend.clone()), + catch_up_targets, + self.checkpoint_interval, + ) + .await; + let primary = BackendEntry { name: "primary".to_string(), role: BackendRole::Primary, - backend: self.primary_backend.clone(), - raw_backend: self.primary_raw_backend.clone(), - operations: Vec::new(), - excludes: Vec::new(), - }); - backends.extend(self.backup_entries); - - let (sync_type, write_ack_count, write_ack_timeout_ms) = match self.sync_mode { - SyncMode::Sync { - ack_count, - timeout_ms, - } => (SyncType::Sync, ack_count, timeout_ms), - SyncMode::Async => (SyncType::Async, usize::MAX, 0), + backend: self.primary_backend, + raw_backend: self.primary_raw_backend, }; - - let write_sem = self - .write_concurrency - .filter(|&n| n > 0) - .map(|n| Arc::new(tokio::sync::Semaphore::new(n))); - - let meta_store = MetaStateStore::new(self.primary_backend, self.ctx_resolver); - - let inner = Arc::new(Inner { - backends, - primary_idx: 0, - sync_type, - write_ack_count, - write_ack_timeout_ms, - write_sem, - redirects: self.redirects, - meta_store, - path_queues: PathSerializer::new(), - pending_dirs: Mutex::new(HashSet::new()), - retry_interval: self.retry_interval, - retry_backoff_base_ms: self.retry_backoff_base_ms, - max_retry_per_round: self.max_retry_per_round, - quarantine_after_failures: self.quarantine_after_failures, - background_tasks: AtomicUsize::new(0), - idle_notify: Notify::new(), - read_backup_hits: AtomicU64::new(0), - read_primary_hits: AtomicU64::new(0), - read_redirect_hits: AtomicU64::new(0), - read_misses: AtomicU64::new(0), - retry_cancelled: AtomicBool::new(false), - retry_shutdown: Notify::new(), - }); - - // Start retry_loop if there are write-enabled backups. - if inner.write_backups().next().is_some() { - inner.background_task_started(); - tokio::spawn(Inner::retry_loop(Arc::clone(&inner))); + let fs = MultiWriteWrappedFS { + inner: Arc::new(Inner { + primary, + runtime, + ctx_resolver: self.ctx_resolver, + }), + metadata_store: metadata_store.clone(), + metadata_provider: metadata_provider.clone(), + }; + if protocol.status != ProtocolStatus::Stable { + let importer_primary: Arc = Arc::new(fs.clone()); + let importer = FullDataImporter::new( + importer_primary, + metadata_store, + metadata_provider, + self.initial_partitions, + backup_names, + ); + fs.inner.runtime.set_import_worker(async move { + let result = importer.run().await; + if let Err(error) = &result { + tracing::error!(error = %error, "multi-write full-data import failed"); + } + result + }); } - - Ok(MultiWriteWrappedFS { inner }) + Ok(fs) } } impl MultiWriteWrappedFS { - /// Return this wrapper's current background task count without scanning metadata. - pub(crate) fn background_task_count(&self) -> usize { - self.inner.background_tasks.load(Ordering::SeqCst) - } - - /// Start building a multi-write wrapper from a primary backend. + /// Start building a wrapper around the supplied primary filesystem. pub fn builder(primary_backend: Arc) -> MultiWriteWrappedFSBuilder { MultiWriteWrappedFSBuilder { primary_backend, primary_raw_backend: None, backup_entries: Vec::new(), - redirects: Vec::new(), - sync_mode: SyncMode::Async, - write_concurrency: None, - retry_interval: DEFAULT_RETRY_INTERVAL, - retry_backoff_base_ms: DEFAULT_RETRY_BACKOFF_BASE_MS, - max_retry_per_round: DEFAULT_MAX_RETRIES_PER_ROUND, - quarantine_after_failures: DEFAULT_QUARANTINE_AFTER_FAILURES, + metadata_store: None, + metadata_provider: None, + initial_partitions: 16, + checkpoint_interval: Duration::from_secs(86_400), ctx_resolver: Arc::new(DefaultFsContextResolver), } } + /// Release prepared runtime work after the wrapper becomes reachable. + pub(crate) fn activate(&self) { + self.inner.runtime.activate(); + } + /// Return the raw primary backend for mount-level internal-name operations. pub(crate) fn primary_raw_backend(&self) -> Option> { - self.inner.primary().raw_backend.clone() + self.inner.primary.raw_backend.clone() } - /// Return whether encrypted backends own dual-path PathLock acquisition. + /// Return whether the primary encryption wrapper owns PathLock acquisition. pub(crate) fn encryption_handles_pathlock(&self) -> bool { - let any = self.inner.primary().backend.as_ref() as &dyn std::any::Any; + let any = self.inner.primary.backend.as_ref() as &dyn std::any::Any; any.downcast_ref::().is_some() } -} - -impl Inner { - /// Build the per-path/per-backend queue key used by both fanout and retry. - fn backup_queue_key(path: &str, backup_name: &str) -> String { - format!("{}\0{}", path, backup_name) - } - - /// Iterate over write-enabled backup entries. - fn write_backups(&self) -> impl Iterator { - self.backends[self.primary_idx + 1..] - .iter() - .filter(|be| be.participates_in_write()) - } - - /// Resolve write-enabled backup targets after applying exclude policies. - fn write_targets(&self, path: &str, size: u64) -> Vec { - self.write_backups() - .filter(|be| !self.is_excluded(be, path, size)) - .map(BackendEntry::fanout_target) - .collect() - } - - /// Resolve explicitly named backup targets. - fn named_targets(&self, target_names: &[String]) -> Vec { - target_names - .iter() - .filter_map(|name| self.backup_by_name(name)) - .map(BackendEntry::fanout_target) - .collect() - } - - /// Execute one backup write synchronously so redirect visibility only appears - /// after at least one target has durably materialized the file contents. - async fn write_first_target( - inner: &Arc, - path: &str, - target: &FanoutTarget, - ctx: &FsContext, - op: BackupWriteOp, - ) -> Result<()> { - let queue_key = Self::backup_queue_key(path, &target.name); - let fs = target.backend.clone(); - let ack_ctx = ctx.clone(); - - inner - .path_queues - .with_path_lock(&queue_key, || async move { - op.apply(inner, fs, path, ctx).await - }) - .await?; - - inner - .update_backup_acked_seq(path, &target.name, &ack_ctx) - .await - } - - /// Resolve effective target backend names for sync/retry work. - pub(crate) fn target_backend_names( - &self, - redirect_meta: &super::types::RedirectMeta, - file_name: &str, - file_path: &str, - sync_entry: &SyncLogEntry, - ) -> Vec { - if let Some(redir) = redirect_meta.entries.get(file_name) { - return redir.targets.clone(); - } - let policy_size = self.retry_policy_size(sync_entry); - self.write_backups() - .filter(|be| !self.is_excluded(be, file_path, policy_size)) - .map(|be| be.name.clone()) - .collect() - } - /// Iterate over read-enabled backup entries sorted by priority. - fn read_backups_sorted(&self) -> Vec<&BackendEntry> { - let mut read_backups: Vec<&BackendEntry> = self.backends[self.primary_idx + 1..] - .iter() - .filter(|be| be.participates_in_read()) - .collect(); - read_backups.sort_by_key(|be| be.read_priority().unwrap_or(u32::MAX)); - read_backups - } - - /// Get the primary backend entry. - pub(crate) fn primary(&self) -> &BackendEntry { - &self.backends[self.primary_idx] - } - - /// Get a backup entry by name. - fn backup_by_name(&self, name: &str) -> Option<&BackendEntry> { - self.backends.iter().find(|be| be.name == name) - } - - /// Check if a file should be excluded from a backup. - fn is_excluded(&self, backup: &BackendEntry, path: &str, size: u64) -> bool { - backup - .excludes - .iter() - .any(|policy| policy.matches(path, size)) - } - - /// Check if a file matches any redirect policy. - fn check_redirect(&self, path: &str, size: u64) -> Option> { - for policy in &self.redirects { - if policy.matches(path, size) { - let targets = match policy { - RedirectPolicy::FileOverSizePolicy { target, .. } => target.clone(), - RedirectPolicy::FileExtensionPolicy { target, .. } => target.clone(), - }; - return targets; - } - } - None - } - - /// Resolve persisted redirect targets for one already-written file path. - async fn redirect_targets_for_path( - &self, - path: &str, - ctx: &FsContext, - ) -> Result>> { - if self.redirects.is_empty() { - return Ok(None); - } - - let dir = parent_dir(path); - let name = file_name(path); - let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?; - Ok(redirect_meta - .entries - .get(name) - .map(|entry| entry.targets.clone())) - } - - /// Generate and persist the next sequence number. - async fn next_seq(&self) -> Result { - self.meta_store.next_seq().await - } - - /// Resolve a file size for retry-time policy decisions. - fn retry_policy_size(&self, sync_entry: &SyncLogEntry) -> u64 { - match &sync_entry.op { - SyncOp::SyncFile { size } => *size, - _ => 0, - } - } - - /// Execute the primary-write, sync-log and backup-fanout pipeline. - async fn execute_write( - inner: &Arc, - path: String, - size: u64, - sync_op: Option, - backup_op: Option, - primary_fn: P, - ) -> Result - where - P: FnOnce(Arc) -> PFut + Send, - PFut: Future> + Send, - { - let ctx = current_required_ctx()?; - inner.invalidate_read_route(&path).await; - - let prepared_entry = if let Some(entry) = sync_op { - let dir = parent_dir(&path); - let name = file_name(&path).to_string(); - let seq = inner.next_seq().await?; - inner - .meta_store - .update_dir_meta(&dir, &ctx, { - let name = name.clone(); - move |_redirect, sync_log| { - sync_log.entries.insert(name, SyncLogEntry::new(seq, entry)); - Ok(()) - } - }) - .await?; - Some((dir, name, seq)) - } else { - None - }; - - let result = match FS_CTX - .scope(ctx.clone(), primary_fn(inner.primary().backend.clone())) - .await - { - Ok(result) => result, - Err(err) => { - if let Some((dir, name, seq)) = prepared_entry.as_ref() { - let _ = inner - .meta_store - .update_dir_meta(dir, &ctx, { - let name = name.clone(); - let seq = *seq; - move |_redirect, sync_log| { - let should_remove = - sync_log.entries.get(&name).is_some_and(|entry| { - entry.latest_seq == seq && !entry.is_primary_committed() - }); - if should_remove { - sync_log.entries.remove(&name); - } - Ok(()) - } - }) - .await; + /// Return one while the flush worker is active and zero after it exits. + pub(crate) fn background_task_count(&self) -> usize { + usize::from(self.inner.runtime.is_running()) + } + + /// Read and validate one complete V2 multi-write metrics snapshot. + pub(crate) async fn metrics(&self) -> Result> { + let protocol_path = self.metadata_store.paths().mount_protocol().0; + let protocol: ProtocolState = self.metadata_store.read_json(&protocol_path).await?; + protocol.validate()?; + let mut lag = BTreeMap::<(String, u32), u64>::new(); + for account in self.metadata_store.initialized_accounts().await? { + let path = self.metadata_store.paths().account_manifest(&account)?.0; + let manifest: PartitionsManifest = self.metadata_store.read_json(&path).await?; + manifest.validate()?; + for (&partition_id, entry) in &manifest.partitions { + if entry.state != PartitionState::Stable { + continue; } - return Err(err); - } - }; - - if let Some((dir, name, seq)) = prepared_entry { - let mut commit_superseded = false; - inner - .meta_store - .update_dir_meta(&dir, &ctx, |_redirect, sync_log| { - let entry = sync_log.entries.get_mut(&name).ok_or_else(|| { - Error::internal(format!( - "prepared sync log entry missing while committing '{}'", - name - )) - })?; - if entry.latest_seq != seq { - warn!( - path = %path, - name = %name, - expected_seq = seq, - actual_seq = entry.latest_seq, - "prepared sync log entry was superseded while committing; skip stale backup fanout" - ); - commit_superseded = true; - return Ok(()); - } - entry.mark_primary_committed(); - Ok(()) - }) - .await?; - if commit_superseded { - inner.refresh_pending_dir(&dir, &ctx).await?; - return Ok(result); - } - inner.mark_pending_dir(&dir).await; - } - - let dir = parent_dir(&path); - let fanout_result = if let Some(backup_op) = backup_op { - Inner::fanout_write(inner, &path, size, ctx.clone(), backup_op).await - } else { - Ok(()) - }; - inner.refresh_pending_dir(&dir, &ctx).await?; - fanout_result?; - Ok(result) - } - - /// Execute a write that may be redirected away from the primary backend. - async fn execute_write_with_redirect( - inner: &Arc, - path: String, - size: u64, - sync_op: SyncOp, - backup_op: BackupWriteOp, - primary_fn: P, - ) -> Result - where - P: FnOnce(Arc) -> PFut + Send, - PFut: Future> + Send, - { - let ctx = current_required_ctx()?; - inner.invalidate_read_route(&path).await; - - if let Some(targets) = inner.check_redirect(&path, size) { - let dir = parent_dir(&path); - let name = file_name(&path).to_string(); - let seq = inner.next_seq().await?; - let resolved_targets = inner.named_targets(&targets); - let first_target = resolved_targets.first().cloned().ok_or_else(|| { - Error::internal(format!( - "redirect path '{}' resolved no writable targets", - path - )) - })?; - Inner::write_first_target(inner, &path, &first_target, &ctx, backup_op.clone()).await?; - let targets_clone = targets.clone(); - let first_target_name = first_target.name.clone(); - inner - .meta_store - .update_dir_meta(&dir, &ctx, move |redirect, sync_log| { - redirect.entries.insert( - name.clone(), - RedirectEntry { - targets: targets_clone.clone(), - }, - ); - let mut committed = SyncLogEntry::committed(seq, sync_op); - committed - .backends - .insert(first_target_name.clone(), BackendSyncState::acked(seq)); - sync_log.entries.insert(name, committed); - Ok(()) - }) - .await?; - - let remaining_targets: Vec = - resolved_targets.into_iter().skip(1).collect(); - if !remaining_targets.is_empty() { - inner.mark_pending_dir(&dir).await; - Inner::fanout_async(inner, &path, remaining_targets, &ctx, backup_op).await; - } - inner.refresh_pending_dir(&dir, &ctx).await?; - return Ok(size); - } - - Inner::execute_write( - inner, - path, - size, - Some(sync_op), - Some(backup_op), - primary_fn, - ) - .await - } - - /// Fanout a write operation to all write-enabled backups. - /// Takes `&Arc` so spawned tasks can clone the Arc for acked_seq updates. - /// `ctx` is required for encrypted backup backends and acked_seq updates. - async fn fanout_write( - inner: &Arc, - path: &str, - size: u64, - ctx: FsContext, - op: BackupWriteOp, - ) -> Result<()> { - Inner::fanout_targets(inner, path, inner.write_targets(path, size), ctx, op).await - } - - /// Fanout a write operation to explicitly named backup targets (used by redirect path). - /// Resolves names to BackendEntry references, then delegates to sync/async state machine. - async fn fanout_write_to_targets( - inner: &Arc, - path: &str, - target_names: &[String], - ctx: FsContext, - op: BackupWriteOp, - ) -> Result<()> { - Inner::fanout_targets(inner, path, inner.named_targets(target_names), ctx, op).await - } - - /// Fanout to already resolved targets using the configured sync mode. - async fn fanout_targets( - inner: &Arc, - path: &str, - targets: Vec, - ctx: FsContext, - op: BackupWriteOp, - ) -> Result<()> { - if targets.is_empty() { - if matches!(inner.sync_type, SyncType::Sync) - && inner.write_ack_count > 0 - && inner.write_backups().next().is_some() - { - return Err(Error::SyncWriteQuorum { - succeeded: 0, - required: inner.write_ack_count, - attempted: 0, - failures: Vec::new(), - }); - } - return Ok(()); - } - - match inner.sync_type { - SyncType::Sync => Inner::fanout_sync(inner, path, &targets, &ctx, op).await, - SyncType::Async => { - Inner::fanout_async(inner, path, targets, &ctx, op).await; - Ok(()) - } - } - } - - /// Synchronous fanout: execute writes in parallel, wait for quorum. - async fn fanout_sync( - inner: &Arc, - path: &str, - targets: &[FanoutTarget], - ctx: &FsContext, - op: BackupWriteOp, - ) -> Result<()> { - let ack_count = inner.write_ack_count.min(targets.len()); - let timeout = if inner.write_ack_timeout_ms > 0 { - Some(Duration::from_millis(inner.write_ack_timeout_ms)) - } else { - None - }; - - let path_owned = path.to_string(); - let ctx = Some(ctx.clone()); - - // Launch parallel tasks for all backup writes. - let mut handles = Vec::new(); - for target in targets { - let fs = target.backend.clone(); - let name = target.name.clone(); - let path = path_owned.clone(); - let inner = Arc::clone(inner); - let ctx = ctx.clone(); - let op_clone = op.clone(); - let queue_key = Self::backup_queue_key(&path, &name); - - handles.push(tokio::spawn(async move { - // Wrap in FS_CTX.scope so encrypted backends can access account_id. - let exec = inner.path_queues.with_path_lock(&queue_key, || async { - op_clone - .apply(&inner, fs, &path, ctx.as_ref().unwrap()) - .await - }); - - let result = if let Some(timeout) = timeout { - match tokio::time::timeout(timeout, exec).await { - Ok(Ok(())) => Ok(()), - Ok(Err(e)) => Err(e), - Err(_) => Err(Error::timeout(format!( - "backup '{}' timed out waiting for sync acknowledgement", - name - ))), - } - } else { - exec.await + let scope = crate::multibackend::model::ScopeKey { + account_id: account.clone(), + partition_id, + epoch: manifest.epoch, }; - - // Update acked_seq on success. - if result.is_ok() { - if let Some(ref ctx) = ctx { - let _ = inner.update_backup_acked_seq(&path, &name, ctx).await; - } - } - - SyncFanoutTaskResult { - backend: name, - result, - } - })); - } - - let results = futures::future::join_all(handles).await; - - let mut successes = 0usize; - let mut failures = Vec::new(); - - for result in results { - match result { - Ok(SyncFanoutTaskResult { - backend: _, - result: Ok(()), - }) => { - successes += 1; - } - Ok(SyncFanoutTaskResult { - backend, - result: Err(e), - }) => { - failures.push(super::errors::SyncWriteFailureDetail { - backend, - kind: e.kind_name().to_string(), - message: e.to_string(), - }); - } - Err(e) => { - failures.push(super::errors::SyncWriteFailureDetail { - backend: "".to_string(), - kind: "join_error".to_string(), - message: e.to_string(), - }); + let segments = self.metadata_provider.read_manifest(&scope).await?; + segments.validate()?; + let tail = segments.next_seq - 1; + for (backend, state) in segments.backend_states { + *lag.entry((backend, partition_id)).or_default() += tail - state.synced_seq; } } } - - if successes >= ack_count { - Ok(()) - } else { - Err(Error::SyncWriteQuorum { - succeeded: successes, - required: ack_count, - attempted: targets.len(), - failures, - }) - } - } - - /// Asynchronous fanout: spawn background tasks that update acked_seq on completion. - /// Uses per-path serialization to prevent out-of-order application on backup backends. - async fn fanout_async( - inner: &Arc, - path: &str, - targets: Vec, - ctx: &FsContext, - op: BackupWriteOp, - ) { - let path_owned = path.to_string(); - let sem = inner.write_sem.clone(); - - for target in targets { - let fs = target.backend.clone(); - let name = target.name.clone(); - let path = path_owned.clone(); - let ctx = ctx.clone(); - let sem = sem.clone(); - let inner = Arc::clone(inner); - let op_clone = op.clone(); - let queue_key = Self::backup_queue_key(&path, &name); - inner.background_task_started(); - - tokio::spawn(async move { - { - // Per (path, backup) serialization preserves FIFO without blocking other backups. - let result = inner - .path_queues - .with_path_lock(&queue_key, || async { - let _permit = if let Some(ref sem) = sem { - sem.acquire().await.ok() - } else { - None - }; - - op_clone.apply(&inner, fs, &path, &ctx).await - }) - .await; - - // Update acked_seq on successful write. - if result.is_ok() { - let _ = inner.update_backup_acked_seq(&path, &name, &ctx).await; - } - } - inner.background_task_finished(); + let mut metrics = vec![ + RagfsMetric { + name: "ragfs_multiwrite_background_tasks".into(), + labels: BTreeMap::new(), + value: RagfsMetricValue::Gauge(self.background_task_count() as f64), + }, + RagfsMetric { + name: "ragfs_multiwrite_pending_events".into(), + labels: BTreeMap::new(), + value: RagfsMetricValue::Gauge(self.inner.runtime.pending_event_count() as f64), + }, + ]; + for (worker, label) in [ + (MultiWriteWorker::Flush, "flush"), + (MultiWriteWorker::Checkpoint, "checkpoint"), + (MultiWriteWorker::CatchUp, "catch_up"), + (MultiWriteWorker::Gc, "gc"), + ] { + metrics.push(RagfsMetric::counter( + "ragfs_multiwrite_errors_total", + &[("worker", label)], + self.metadata_store.worker_error_count(worker), + 1.0, + )); + } + metrics.push(RagfsMetric { + name: "ragfs_multiwrite_protocol_version".into(), + labels: BTreeMap::from([("version".into(), "v2".into())]), + value: RagfsMetricValue::Gauge(u64::from( + protocol.protocol_version == ProtocolVersion::V2, + ) as f64), + }); + for (status, value) in [ + ( + "migrating", + u64::from(protocol.status == ProtocolStatus::Migrating), + ), + ( + "stable", + u64::from(protocol.status == ProtocolStatus::Stable), + ), + ] { + metrics.push(RagfsMetric { + name: "ragfs_multiwrite_protocol_status".into(), + labels: BTreeMap::from([("status".into(), status.into())]), + value: RagfsMetricValue::Gauge(value as f64), }); } + metrics.extend( + lag.into_iter() + .map(|((backend, partition), value)| RagfsMetric { + name: "ragfs_multiwrite_backend_lag_events".into(), + labels: BTreeMap::from([ + ("backend".into(), backend), + ("partition".into(), partition.to_string()), + ]), + value: RagfsMetricValue::Gauge(value as f64), + }), + ); + Ok(metrics) + } + + /// Resolve one account context and reject disagreement between path and task. + fn resolve_context(&self, path: &str) -> Result { + let path_ctx = self.inner.ctx_resolver.resolve(path)?; + match current_required_ctx() { + Ok(ctx) if !ctx.account_id().trim().is_empty() => { + if ctx.account_id() != path_ctx.account_id() { + return Err(Error::invalid_path(format!( + "path account '{}' conflicts with context account '{}'", + path_ctx.account_id(), + ctx.account_id() + ))); + } + Ok(ctx) + } + _ => Ok(path_ctx), + } } - /// Record rename metadata and migrate redirect state when needed. - async fn record_rename_meta( + /// Build one canonical V2 event from a foreground mount-relative path. + fn pending_event( &self, - old_path: &str, - new_path: &str, - ctx: &FsContext, - ) -> Result<()> { - let source_dir = parent_dir(old_path); - let target_dir = parent_dir(new_path); - let old_name = file_name(old_path).to_string(); - let new_name = file_name(new_path).to_string(); - let seq = self.next_seq().await?; - let rename_op = SyncOp::Rename { - to: new_path.to_string(), - }; - - if source_dir == target_dir { - self.meta_store - .update_dir_meta(&source_dir, ctx, move |redirect, sync_log| { - sync_log - .entries - .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op)); - if let Some(redirect_entry) = redirect.entries.remove(&old_name) { - redirect.entries.insert(new_name, redirect_entry); - } - Ok(()) - }) - .await - } else { - self.meta_store - .update_dual_dir_meta( - &source_dir, - &target_dir, - ctx, - move |src_redirect, src_sync_log, tgt_redirect, _tgt_sync_log| { - src_sync_log - .entries - .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op)); - if let Some(redirect_entry) = src_redirect.entries.remove(&old_name) { - tgt_redirect.entries.insert(new_name, redirect_entry); - } - Ok(()) - }, - ) - .await + path: &str, + kind: SegmentEventType, + destination_path: Option<&str>, + ) -> Result> { + let ctx = self.resolve_context(path)?; + let account_id = ctx.account_id().to_string(); + let mut path = logical_path(&account_id, path)?; + let mut destination_path = destination_path + .map(|destination| logical_path(&account_id, destination)) + .transpose()?; + let mut kind = kind; + let source_internal = is_multiwrite_internal_path(&path); + let destination_internal = destination_path + .as_deref() + .is_some_and(is_multiwrite_internal_path); + match (kind, source_internal, destination_internal) { + (SegmentEventType::MoveTree, true, false) => { + path = destination_path + .take() + .ok_or_else(|| Error::invalid_operation("move event requires a destination"))?; + kind = SegmentEventType::RemoveTree; + } + (SegmentEventType::MoveTree, false, true) => { + kind = SegmentEventType::RemoveTree; + destination_path = None; + } + (_, true, _) | (_, _, true) => return Ok(None), + _ => {} } + Ok(Some(PendingEvent { + operation_id: Uuid::new_v4(), + account_id, + kind: PendingEventKind::Data(kind), + path, + destination_path, + route_hint: None, + })) } -} - -/// Return true when `path` falls under `exclude_path` (including itself). -fn is_excluded_grep_path(path: &str, exclude_path: Option<&str>) -> bool { - let Some(exclude_path) = exclude_path.map(normalize_prefix_path) else { - return false; - }; - let normalized_path = normalize_prefix_path(path); - normalized_path == exclude_path - || normalized_path - .strip_prefix(&exclude_path) - .is_some_and(|suffix| suffix.starts_with('/')) -} - -// ── FileSystem trait implementation ── - -impl Drop for MultiWriteWrappedFS { - /// Signal retry_loop to exit when the wrapper is unmounted or dropped. - fn drop(&mut self) { - self.inner.retry_cancelled.store(true, Ordering::SeqCst); - self.inner.retry_shutdown.notify_waiters(); - } -} -impl MultiWriteWrappedFS { - /// Stop background retry work and wait for in-flight async fanout to drain. - pub async fn shutdown(&self) -> Result<()> { - self.inner.retry_cancelled.store(true, Ordering::SeqCst); - self.inner.retry_shutdown.notify_waiters(); - self.inner.wait_idle(DEFAULT_SHUTDOWN_WAIT).await + /// Submit one prebuilt event after its primary operation has succeeded. + fn submit_event(&self, event: Option) { + if let Some(event) = event { + self.inner.runtime.submit(event); + } } - /// Copy one primary-resident file to a new path while preserving physical primary bytes. + /// Copy raw primary bytes to another primary path and queue one write event. pub async fn copy_within_primary(&self, src_path: &str, dst_path: &str) -> Result { if normalize_prefix_path(src_path) == normalize_prefix_path(dst_path) { return Ok(true); } - - let ctx = current_required_ctx()?; - let inner = &self.inner; - let source_path = src_path.to_string(); - let destination_path = dst_path.to_string(); - - let source_exists_on_primary = FS_CTX - .scope(ctx.clone(), async { - inner.primary().backend.exists(&source_path).await - }) - .await; - if !source_exists_on_primary { + let Some(primary_raw) = self.inner.primary.raw_backend.clone() else { return Ok(false); - } - - let source_size = FS_CTX - .scope(ctx.clone(), async { - let source_info = inner.primary().backend.stat(&source_path).await?; - if source_info.is_dir { - return Err(Error::IsADirectory(source_path.clone())); - } - Ok::(source_info.size) - }) - .await?; - - if inner - .check_redirect(&destination_path, source_size) - .is_some() - { + }; + if !primary_raw.exists(src_path).await { return Ok(false); } - - let Some(primary_raw_backend) = inner.primary().raw_backend.clone() else { - return Ok(false); - }; - - let primary_destination = destination_path.clone(); - Inner::execute_write( - inner, - destination_path, - source_size, - Some(SyncOp::SyncFile { size: source_size }), - Some(BackupWriteOp::Replay(SyncOp::SyncFile { - size: source_size, - })), - move |_ignored_fs| { - let primary_raw_backend = primary_raw_backend.clone(); - let primary_source = source_path.clone(); - let primary_destination = primary_destination.clone(); - let primary_ctx = ctx.clone(); - async move { - copy_raw_primary_state( - primary_raw_backend, - &primary_source, - &primary_destination, - &primary_ctx, - ) - .await?; - Ok(()) - } - }, - ) - .await?; - + let ctx = self.resolve_context(src_path)?; + logical_path(ctx.account_id(), dst_path)?; + let event = self.pending_event(dst_path, SegmentEventType::Write, None)?; + copy_raw_primary_state(primary_raw, src_path, dst_path, &ctx).await?; + self.submit_event(event); Ok(true) } +} - /// Execute one non-redirecting write-like operation through the shared multi-write pipeline. - async fn execute_simple_write( - &self, - path: &str, - size: u64, - sync_op: Option, - primary_fn: P, - ) -> Result - where - R: Send + 'static, - P: FnOnce(Arc, String) -> PFut + Send, - PFut: Future> + Send, +/// Convert a wrapper path into one canonical account-owned logical path. +fn logical_path(account_id: &str, path: &str) -> Result { + let normalized = normalize_prefix_path(path); + let logical = if normalized.starts_with('/') { + format!("/local{normalized}") + } else { + format!("/local/{normalized}") + }; + let account_root = format!("/local/{account_id}"); + if logical != account_root + && !logical + .strip_prefix(&account_root) + .is_some_and(|suffix| suffix.starts_with('/')) { - let path_owned = path.to_string(); - let primary_path = path_owned.clone(); - let backup_op = sync_op.clone().map(BackupWriteOp::Replay); - Inner::execute_write( - &self.inner, - path_owned, - size, - sync_op, - backup_op, - move |fs| primary_fn(fs, primary_path), - ) + return Err(Error::invalid_path(format!( + "path does not belong to account '{account_id}': {logical}" + ))); + } + Ok(logical) +} + +/// Copy one complete raw file in bounded chunks under the request context. +async fn copy_raw_primary_state( + primary_raw: Arc, + source_path: &str, + destination_path: &str, + ctx: &FsContext, +) -> Result<()> { + FS_CTX + .scope(ctx.clone(), async { + let source = primary_raw.stat(source_path).await?; + if source.is_dir { + return Err(Error::IsADirectory(source_path.to_string())); + } + primary_raw + .ensure_parent_dirs(destination_path, 0o755) + .await?; + if source.size == 0 { + if primary_raw.exists(destination_path).await { + return primary_raw.truncate(destination_path, 0).await; + } + return primary_raw.create(destination_path).await; + } + let mut offset = 0; + while offset < source.size { + let size = (source.size - offset).min(COPY_CHUNK_SIZE as u64); + let chunk = primary_raw.read(source_path, offset, size).await?; + if chunk.is_empty() { + return Err(Error::internal(format!( + "short read while copying raw primary file: {source_path}" + ))); + } + let flag = if offset == 0 { + WriteFlag::Create + } else { + WriteFlag::None + }; + primary_raw + .write(destination_path, &chunk, offset, flag) + .await?; + offset += chunk.len() as u64; + } + Ok(()) + }) .await +} + +/// Build the V2 event specs required to represent one completed rename. +fn rename_event_specs<'a>( + source: &FileInfo, + old_path: &'a str, + new_path: &'a str, +) -> Vec<(SegmentEventType, &'a str, Option<&'a str>)> { + if source.is_dir { + vec![(SegmentEventType::MoveTree, old_path, Some(new_path))] + } else { + vec![ + (SegmentEventType::Write, new_path, None), + (SegmentEventType::Remove, old_path, None), + ] } } #[async_trait] impl FileSystem for MultiWriteWrappedFS { + /// Create a primary file and queue a write event after success. async fn create(&self, path: &str) -> Result<()> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.create(path).await; - } - self.execute_simple_write(path, 0, Some(SyncOp::Create), |fs, path| async move { - fs.create(&path).await - }) - .await + let event = self.pending_event(path, SegmentEventType::Write, None)?; + self.inner.primary.backend.create(path).await?; + self.submit_event(event); + Ok(()) } + /// Create a primary directory without producing a file-state event. async fn mkdir(&self, path: &str, mode: u32) -> Result<()> { - self.execute_simple_write( - path, - 0, - Some(SyncOp::Mkdir { mode }), - move |fs, path| async move { fs.mkdir(&path, mode).await }, - ) - .await + self.inner.primary.backend.mkdir(path, mode).await } + /// Remove a primary file and queue a remove event after success. async fn remove(&self, path: &str) -> Result<()> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.remove(path).await; - } - self.execute_simple_write(path, 0, Some(SyncOp::Remove), |fs, path| async move { - fs.remove(&path).await - }) - .await + let event = self.pending_event(path, SegmentEventType::Remove, None)?; + self.inner.primary.backend.remove(path).await?; + self.submit_event(event); + Ok(()) } + /// Remove a primary tree and queue a directory removal event after success. async fn remove_all(&self, path: &str) -> Result<()> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.remove_all(path).await; + let event = self + .pending_event(path, SegmentEventType::RemoveTree, None)? + .map(|mut event| { + if event.path == format!("/local/{}", event.account_id) { + event.kind = PendingEventKind::DeleteAccount; + } + event + }); + let is_delete_account = event + .as_ref() + .is_some_and(|event| matches!(event.kind, PendingEventKind::DeleteAccount)); + match self.inner.primary.backend.remove_all(path).await { + Ok(()) => { + self.submit_event(event); + Ok(()) + } + Err(error @ Error::NotFound(_)) if is_delete_account => { + self.submit_event(event); + Err(error) + } + Err(error) => Err(error), } - self.execute_simple_write(path, 0, Some(SyncOp::RemoveAll), |fs, path| async move { - fs.remove_all(&path).await - }) - .await } + /// Read bytes directly from the primary backend. async fn read(&self, path: &str, offset: u64, size: u64) -> Result> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.read(path, offset, size).await; - } - if let Some(fs) = self.inner.resolve_read_backend(path).await { - return fs.read(path, offset, size).await; - } - Err(Error::not_found(path)) + self.inner.primary.backend.read(path, offset, size).await } + /// Write primary bytes and queue a write event after success. async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result { - if is_hidden_internal_name(file_name(path)) { - return self - .inner - .primary() - .backend - .write(path, data, offset, flags) - .await; - } - let inner = &self.inner; - let data_len = data.len() as u64; - let path_owned = path.to_string(); - let backup_op = BackupWriteOp::WriteFile { - data: Arc::new(data.to_vec()), - offset, - flags, - }; - let d = data.to_vec(); - let primary_path = path_owned.clone(); - Inner::execute_write_with_redirect( - inner, - path_owned, - data_len, - SyncOp::SyncFile { size: data_len }, - backup_op, - move |fs: Arc| async move { - fs.write(&primary_path, &d, offset, flags).await - }, - ) - .await + let event = self.pending_event(path, SegmentEventType::Write, None)?; + let written = self + .inner + .primary + .backend + .write(path, data, offset, flags) + .await?; + self.submit_event(event); + Ok(written) } + /// List one directory directly from the primary backend. async fn read_dir( &self, path: &str, @@ -1418,379 +594,89 @@ impl FileSystem for MultiWriteWrappedFS { sort_by: Option, sort_order: Option, ) -> Result> { - let inner = &self.inner; - let mut entries = inner - .primary() + self.inner + .primary .backend - .read_dir(path, None, None, None, None) - .await?; - - // Filter multi-write internal names. - entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str())); - - if inner.redirects.is_empty() { - return Ok(apply_read_dir_options( - entries, offset, limit, sort_by, sort_order, - )); - } - - // Merge redirect entries so users can see redirected files in listings. - let ctx = - current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?; - - if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(path, &ctx).await { - for (name, redirect_entry) in &redirect_meta.entries { - if !entries.iter().any(|e| &e.name == name) { - let virtual_path = if path == "/" { - format!("/{}", name) - } else { - format!("{}/{}", path.trim_end_matches('/'), name) - }; - entries.push( - inner - .redirect_file_info(&virtual_path, name, redirect_entry) - .await, - ); - } - } - } + .read_dir(path, offset, limit, sort_by, sort_order) + .await + } - Ok(apply_read_dir_options( - entries, offset, limit, sort_by, sort_order, - )) + /// List internal entries directly from the primary backend. + async fn read_internal_dir(&self, path: &str) -> Result> { + self.inner.primary.backend.read_internal_dir(path).await } + /// Read metadata directly from the primary backend. async fn stat(&self, path: &str) -> Result { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.stat(path).await; - } - if let Some(fs) = self.inner.resolve_read_backend(path).await { - return fs.stat(path).await; - } - Err(Error::not_found(path)) + self.inner.primary.backend.stat(path).await } + /// Rename a primary path and queue file or tree events after success. async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> { - if is_hidden_internal_name(file_name(old_path)) - || is_hidden_internal_name(file_name(new_path)) - { - return self - .inner - .primary() - .backend - .rename(old_path, new_path) - .await; - } - let ctx = current_required_ctx()?; - let inner = &self.inner; - let old_owned = old_path.to_string(); - let new_owned = new_path.to_string(); - let source_dir = parent_dir(&old_owned); - let redirect_targets = inner.redirect_targets_for_path(&old_owned, &ctx).await?; - inner.invalidate_read_route(&old_owned).await; - inner.invalidate_read_route(&new_owned).await; - - let primary_has_old = inner.primary().backend.exists(&old_owned).await; - if primary_has_old { - FS_CTX - .scope(ctx.clone(), async { - inner.primary().backend.rename(&old_owned, &new_owned).await - }) - .await?; - } else if redirect_targets.is_none() { - return Err(Error::NotFound(old_owned)); - } - - inner - .record_rename_meta(&old_owned, &new_owned, &ctx) + let old_ctx = self.resolve_context(old_path)?; + let new_ctx = self.resolve_context(new_path)?; + if old_ctx.account_id() != new_ctx.account_id() { + return Err(Error::invalid_path(format!( + "rename paths cross accounts: '{}' -> '{}'", + old_path, new_path + ))); + } + let source = self.inner.primary.backend.stat(old_path).await?; + let events = rename_event_specs(&source, old_path, new_path) + .into_iter() + .map(|(kind, path, destination)| self.pending_event(path, kind, destination)) + .collect::>>()?; + self.inner + .primary + .backend + .rename(old_path, new_path) .await?; - inner.mark_pending_dir(&source_dir).await; - - let fanout_path = old_owned.clone(); - let backup_op = BackupWriteOp::Replay(SyncOp::Rename { - to: new_owned.clone(), - }); - if let Some(targets) = redirect_targets { - Inner::fanout_write_to_targets(inner, &fanout_path, &targets, ctx.clone(), backup_op) - .await?; - } else { - Inner::fanout_write(inner, &fanout_path, 0, ctx.clone(), backup_op).await?; + for event in events { + self.submit_event(event); } - inner.refresh_pending_dir(&source_dir, &ctx).await?; - Ok(()) } + /// Change primary permissions and queue a write event after success. async fn chmod(&self, path: &str, mode: u32) -> Result<()> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.chmod(path, mode).await; - } - self.execute_simple_write( - path, - 0, - Some(SyncOp::Chmod { mode }), - move |fs, path| async move { fs.chmod(&path, mode).await }, - ) - .await + let event = self.pending_event(path, SegmentEventType::Write, None)?; + self.inner.primary.backend.chmod(path, mode).await?; + self.submit_event(event); + Ok(()) } + /// Truncate a primary file and queue a write event after success. async fn truncate(&self, path: &str, size: u64) -> Result<()> { - if is_hidden_internal_name(file_name(path)) { - return self.inner.primary().backend.truncate(path, size).await; - } - self.execute_simple_write( - path, - size, - Some(SyncOp::SyncFile { size }), - move |fs, path| async move { fs.truncate(&path, size).await }, - ) - .await + let event = self.pending_event(path, SegmentEventType::Write, None)?; + self.inner.primary.backend.truncate(path, size).await?; + self.submit_event(event); + Ok(()) } + /// Create primary parent directories without producing a file-state event. async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> { - let path_owned = path.to_string(); - let primary_path = path_owned.clone(); - Inner::execute_write( - &self.inner, - path_owned, - 0, - None, - Some(BackupWriteOp::EnsureParentDirs { mode }), - move |fs| async move { fs.ensure_parent_dirs(&primary_path, mode).await }, - ) - .await + self.inner + .primary + .backend + .ensure_parent_dirs(path, mode) + .await } + /// Search content directly on the primary backend. async fn grep( &self, path: &str, pattern: &str, options: GrepOptions<'_>, ) -> Result { - let inner = &self.inner; - let path_owned = path.to_string(); - let pattern_owned = pattern.to_string(); - let exclude_owned = options.exclude_path.map(str::to_string); - let options = GrepOptions { - exclude_path: exclude_owned.as_deref(), - ..options - }; - let recursive = options.recursive; - let node_limit = options.node_limit; - let level_limit = options.level_limit; - - let mut result = inner - .primary() - .backend - .grep(&path_owned, &pattern_owned, options) - .await?; - - // Filter out multi-write internal metadata files from grep results. - result.matches.retain(|m| { - let file_name = m.file.rsplit('/').next().unwrap_or(&m.file); - !MULTIWRITE_INTERNAL_NAMES.contains(&file_name) - }); - result.count = result.matches.len(); - - if inner.redirects.is_empty() { - return Ok(result); - } - - // For redirect files, also grep in target backends. - let ctx = current_required_ctx() - .or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?; - - let search_dir = if inner - .primary() + self.inner + .primary .backend - .stat(&path_owned) + .grep(path, pattern, options) .await - .map(|s| s.is_dir) - .unwrap_or(false) - { - path_owned.clone() - } else { - parent_dir(&path_owned) - }; - - if recursive && search_dir == path_owned { - let redirect_entries = self - .tree_directory( - &path_owned, - true, - None, - level_limit, - None, - None, - None, - false, - ) - .await?; - for entry in redirect_entries { - if node_limit.is_some_and(|limit| result.count >= limit) { - break; - } - if entry.info.is_dir - || entry.extra.get("redirect").and_then(Value::as_bool) != Some(true) - { - continue; - } - if is_excluded_grep_path(&entry.path, exclude_owned.as_deref()) { - continue; - } - let Some(read_backend) = inner.resolve_read_backend(&entry.path).await else { - continue; - }; - let rel_path = relative_match_file(&path_owned, &entry.path); - let target_result = match read_backend - .grep( - &entry.path, - &pattern_owned, - GrepOptions { - recursive: false, - node_limit: node_limit.map(|limit| limit.saturating_sub(result.count)), - exclude_path: None, - level_limit: None, - ..options - }, - ) - .await - { - Ok(found) => found, - Err(_) => continue, - }; - for mut m in target_result.matches { - if node_limit.is_some_and(|limit| result.count >= limit) { - break; - } - m.file = rel_path.clone(); - result.matches.push(m); - result.count += 1; - } - } - return Ok(result); - } - - if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(&search_dir, &ctx).await { - for (name, redirect_entry) in &redirect_meta.entries { - for target_name in &redirect_entry.targets { - if let Some(be) = inner.backup_by_name(target_name) { - let redirect_path = if search_dir == "/" { - format!("/{}", name) - } else { - format!("{}/{}", search_dir, name) - }; - if let Ok(target_result) = be - .backend - .grep( - &redirect_path, - &pattern_owned, - GrepOptions { - recursive: false, - exclude_path: None, - level_limit: None, - ..options - }, - ) - .await - { - let rel_path = relative_match_file(&path_owned, &redirect_path); - for mut m in target_result.matches { - if node_limit.is_some_and(|limit| result.count >= limit) { - break; - } - m.file = rel_path.clone(); - result.matches.push(m); - result.count += 1; - } - } - } - } - } - } - - Ok(result) - } - - async fn glob_directory( - &self, - path: &str, - pattern: &str, - show_hidden: bool, - page_size: Option, - level_limit: Option, - continuation_token: Option, - ) -> Result { - if self.inner.redirects.is_empty() { - return self - .inner - .primary() - .backend - .glob_directory( - path, - pattern, - show_hidden, - page_size, - level_limit, - continuation_token, - ) - .await; - } - - let matcher = PreparedGlob::new(pattern)?; - if matches!(page_size, Some(0)) { - return Err(Error::invalid_operation("page_size must be positive")); - } - - let entries = self - .tree_directory( - path, - show_hidden, - None, - level_limit, - None, - None, - None, - false, - ) - .await?; - - let mut matched = Vec::new(); - for entry in entries { - if matcher.is_match(&entry.rel_path) { - matched.push(GlobEntry { - path: entry.path, - rel_path: entry.rel_path, - name: entry.info.name, - is_dir: entry.info.is_dir, - }); - } - } - matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path)); - - let start = decode_offset_token( - continuation_token.as_deref(), - path, - pattern, - show_hidden, - level_limit, - )?; - if start > matched.len() { - return Err(Error::invalid_operation("continuation token out of range")); - } - let end = page_size - .map(|limit| start.saturating_add(limit)) - .unwrap_or(matched.len()) - .min(matched.len()); - let next_token = (end < matched.len()) - .then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit)); - - Ok(GlobPage { - entries: matched[start..end].to_vec(), - next_token, - }) } + /// Traverse a tree directly on the primary backend. async fn tree_directory( &self, path: &str, @@ -1802,130 +688,43 @@ impl FileSystem for MultiWriteWrappedFS { sort_order: Option, directories_only: bool, ) -> Result> { - let base = normalize_prefix_path(path); - if sort_by.is_some() { - let mut entries = Vec::new(); - let traversal_limit = node_limit.map(|limit| offset.unwrap_or(0).saturating_add(limit)); - self.tree_directory_internal( - &base, - &base, + self.inner + .primary + .backend + .tree_directory( + path, show_hidden, - traversal_limit, + node_limit, level_limit, + offset, sort_by, sort_order, directories_only, - &mut entries, ) - .await?; - return Ok(paginate_entries(entries, offset, node_limit)); - } + .await + } - let mut entries = self - .inner - .primary() + /// Match paths directly on the primary backend. + async fn glob_directory( + &self, + path: &str, + pattern: &str, + show_hidden: bool, + page_size: Option, + level_limit: Option, + continuation_token: Option, + ) -> Result { + self.inner + .primary .backend - .tree_directory( + .glob_directory( path, + pattern, show_hidden, - None, + page_size, level_limit, - None, - None, - None, - directories_only, + continuation_token, ) - .await?; - - entries.retain(|e| { - let name = file_name(&e.path); - !MULTIWRITE_INTERNAL_NAMES.contains(&name) && (!directories_only || e.info.is_dir) - }); - - if self.inner.redirects.is_empty() { - return Ok(paginate_entries(entries, offset, node_limit)); - } - - let ctx = current_required_ctx() - .or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?; - let mut seen_paths: HashSet = entries.iter().map(|e| e.path.clone()).collect(); - let mut dir_paths = Vec::new(); - let mut queued_dirs: HashSet = HashSet::new(); - if !matches!(level_limit, Some(0)) { - queued_dirs.insert(base.clone()); - dir_paths.push(base.clone()); - } - for entry in &entries { - if entry.info.is_dir { - let dir = normalize_prefix_path(&entry.path); - let may_emit_redirect_children = match level_limit { - Some(limit) => { - let rel = relative_match_file(&base, &dir); - relative_depth(&rel) < limit - } - None => true, - }; - if may_emit_redirect_children && queued_dirs.insert(dir.clone()) { - dir_paths.push(dir); - } - } - } - - let inner = Arc::clone(&self.inner); - let mut redirect_results = stream::iter(dir_paths) - .map(|dir| { - let inner = Arc::clone(&inner); - let ctx = ctx.clone(); - async move { - let redirect_meta = inner.meta_store.get_redirect_meta(&dir, &ctx).await.ok(); - (dir, redirect_meta) - } - }) - .buffered(DEFAULT_TREE_REDIRECT_META_CONCURRENCY); - - while let Some((dir, redirect_meta)) = redirect_results.next().await { - let Some(redirect_meta) = redirect_meta else { - continue; - }; - for (name, redirect_entry) in redirect_meta.entries { - let virtual_path = if dir == "/" { - format!("/{}", name) - } else { - format!("{}/{}", dir, name) - }; - if seen_paths.contains(&virtual_path) { - continue; - } - let rel_path = if base == "/" { - virtual_path.trim_start_matches('/').to_string() - } else { - virtual_path - .strip_prefix(&base) - .unwrap_or(&virtual_path) - .trim_start_matches('/') - .to_string() - }; - if level_limit.is_some_and(|limit| relative_depth(&rel_path) > limit) { - continue; - } - let mut extra = HashMap::new(); - extra.insert("redirect".to_string(), Value::Bool(true)); - entries.push(TreeEntry { - path: virtual_path.clone(), - rel_path, - info: self - .inner - .redirect_file_info(&virtual_path, &name, &redirect_entry) - .await, - extra, - }); - seen_paths.insert(virtual_path); - } - } - - if directories_only { - entries.retain(|entry| entry.info.is_dir); - } - Ok(paginate_entries(entries, offset, node_limit)) + .await } } diff --git a/crates/ragfs/src/core/multibackend_wrapper/retry.rs b/crates/ragfs/src/core/multibackend_wrapper/retry.rs deleted file mode 100644 index c4581bf227..0000000000 --- a/crates/ragfs/src/core/multibackend_wrapper/retry.rs +++ /dev/null @@ -1,270 +0,0 @@ -use std::time::{Duration, Instant}; - -use rand::Rng; -use tracing::warn; - -use super::*; - -impl Inner { - /// Mark one directory as pending retry work. - pub(super) async fn mark_pending_dir(&self, dir: &str) { - self.pending_dirs.lock().await.insert(dir.to_string()); - } - - /// Remove one directory from the retry set. - async fn clear_pending_dir(&self, dir: &str) { - self.pending_dirs.lock().await.remove(dir); - } - - /// Snapshot the current set of pending retry directories. - pub(crate) async fn pending_dirs_snapshot(&self) -> Vec { - self.pending_dirs.lock().await.iter().cloned().collect() - } - - /// Recompute whether one directory still has pending retry work. - pub(super) async fn refresh_pending_dir(&self, dir: &str, ctx: &FsContext) -> Result<()> { - if self.sync_work_in_dir(dir, ctx).await?.iter().any(|work| { - work.entry.is_primary_committed() - && work.targets.iter().any(|target| { - !work.entry.is_in_sync(target) && !work.entry.is_quarantined(target) - }) - }) { - self.mark_pending_dir(dir).await; - } else { - self.clear_pending_dir(dir).await; - } - Ok(()) - } - - /// Collect sync work entries for one metadata directory. - pub(crate) async fn sync_work_in_dir( - &self, - dir: &str, - ctx: &FsContext, - ) -> Result> { - let sync_log = self.meta_store.get_sync_log_meta(dir, ctx).await?; - if sync_log.entries.is_empty() { - return Ok(Vec::new()); - } - - let redirect_meta = self.meta_store.get_redirect_meta(dir, ctx).await?; - Ok(sync_log - .entries - .into_iter() - .map(|(file_name, entry)| { - let file_path = if dir == "/" { - format!("/{}", file_name) - } else { - format!("{}/{}", dir, file_name) - }; - let targets = - self.target_backend_names(&redirect_meta, &file_name, &file_path, &entry); - SyncWorkEntry { - file_path, - entry, - targets, - } - }) - .collect()) - } - - /// Record a started background task. - pub(super) fn background_task_started(&self) { - self.background_tasks.fetch_add(1, Ordering::SeqCst); - } - - /// Record a finished background task and wake idle waiters if needed. - pub(super) fn background_task_finished(&self) { - if self.background_tasks.fetch_sub(1, Ordering::SeqCst) == 1 { - self.idle_notify.notify_waiters(); - } - } - - /// Wait for all background tasks to drain. - pub(super) async fn wait_idle(&self, timeout: Duration) -> Result<()> { - let deadline = Instant::now() + timeout; - loop { - if self.background_tasks.load(Ordering::SeqCst) == 0 { - return Ok(()); - } - let now = Instant::now(); - if now >= deadline { - return Err(Error::timeout( - "timed out while waiting multi-write background tasks to drain", - )); - } - let wait = deadline.saturating_duration_since(now); - tokio::time::timeout(wait, self.idle_notify.notified()) - .await - .map_err(|_| { - Error::timeout("timed out while waiting multi-write background tasks to drain") - })?; - } - } - - /// Update the acked_seq for a backup in the sync log. - pub(super) async fn update_backup_acked_seq( - &self, - path: &str, - backup_name: &str, - ctx: &FsContext, - ) -> Result<()> { - self.update_backend_state(path, backup_name, ctx, |state, latest_seq| { - state.mark_acked(latest_seq); - }) - .await?; - self.refresh_pending_dir(&parent_dir(path), ctx).await - } - - /// Record a replay failure and quarantine the target after repeated failures. - pub(crate) async fn record_backup_retry_failure( - &self, - path: &str, - backup_name: &str, - ctx: &FsContext, - ) -> Result<()> { - self.update_backend_state(path, backup_name, ctx, |state, _latest_seq| { - state.mark_retry_failed(self.quarantine_after_failures); - }) - .await?; - self.refresh_pending_dir(&parent_dir(path), ctx).await - } - - /// Update per-backend sync state for a file entry. - async fn update_backend_state( - &self, - path: &str, - backup_name: &str, - ctx: &FsContext, - update: F, - ) -> Result<()> - where - F: FnOnce(&mut BackendSyncState, u64) + Send, - { - let dir = parent_dir(path); - let backup_name = backup_name.to_string(); - let name = file_name(path).to_string(); - self.meta_store - .update_dir_meta(&dir, ctx, move |_redirect, sync_log| { - if let Some(entry) = sync_log.entries.get_mut(&name) { - let state = entry.backends.entry(backup_name).or_default(); - update(state, entry.latest_seq); - } - Ok(()) - }) - .await - } - - /// Replay a single operation on a lagging backup. - pub(crate) async fn replay_operation( - &self, - file_path: &str, - backup_name: &str, - ctx: &FsContext, - ) -> Result<()> { - let queue_key = Self::backup_queue_key(file_path, backup_name); - self.path_queues - .with_path_lock(&queue_key, || async { - let dir = parent_dir(file_path); - let name = file_name(file_path).to_string(); - let sync_log = self.meta_store.get_sync_log_meta(&dir, ctx).await?; - let entry = sync_log - .entries - .get(&name) - .ok_or_else(|| Error::not_found(file_path))? - .clone(); - let backup = match self.backup_by_name(backup_name) { - Some(b) => b, - None => { - return Err(Error::internal(format!( - "backup '{}' not found", - backup_name - ))) - } - }; - - entry - .op - .replay( - self.primary().backend.clone(), - backup.backend.clone(), - file_path, - ctx, - ) - .await?; - - self.update_backup_acked_seq(file_path, backup_name, ctx) - .await?; - - Ok(()) - }) - .await - } - - /// Retry all pending backup work for one directory and keep metadata errors visible. - pub(crate) async fn retry_pending_dir(&self, dir: &str) -> Result<()> { - let ctx = self.meta_store.ctx_resolver().resolve(dir)?; - - for work in self.sync_work_in_dir(dir, &ctx).await? { - if !work.entry.is_primary_committed() { - continue; - } - - for backup_name in &work.targets { - if work.entry.is_in_sync(backup_name) || work.entry.is_quarantined(backup_name) { - continue; - } - - let mut success = false; - for attempt in 0..self.max_retry_per_round { - if self.retry_cancelled.load(Ordering::SeqCst) { - return Ok(()); - } - if self - .replay_operation(&work.file_path, backup_name, &ctx) - .await - .is_ok() - { - success = true; - break; - } - let base_ms = self.retry_backoff_base_ms.saturating_mul(1u64 << attempt); - let jitter_ms = rand::thread_rng().gen_range(0..=50); - tokio::time::sleep(Duration::from_millis(base_ms + jitter_ms)).await; - } - - if !success { - self.record_backup_retry_failure(&work.file_path, backup_name, &ctx) - .await?; - } - } - } - - self.refresh_pending_dir(dir, &ctx).await - } - - /// Background retry loop: periodically scans sync_log for lagging backups and replays. - pub(super) async fn retry_loop(inner: Arc) { - loop { - if inner.retry_cancelled.load(Ordering::SeqCst) { - break; - } - - tokio::select! { - _ = tokio::time::sleep(inner.retry_interval) => {} - _ = inner.retry_shutdown.notified() => break, - } - let dirs = inner.pending_dirs_snapshot().await; - for dir in dirs { - if let Err(err) = inner.retry_pending_dir(&dir).await { - warn!( - dir = %dir, - error = %err, - "multi-write retry skipped one pending directory due to metadata or replay error" - ); - } - } - } - inner.background_task_finished(); - } -} diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rs deleted file mode 100644 index 1dff3f491b..0000000000 --- a/crates/ragfs/src/core/multibackend_wrapper/routing.rs +++ /dev/null @@ -1,125 +0,0 @@ -use std::sync::Arc; - -use serde_json::{json, Value}; - -use super::*; -use crate::multibackend::meta::{current_required_ctx, file_name, parent_dir}; - -impl Inner { - /// Keep write-side call sites explicit even though read routing is uncached. - pub(super) async fn invalidate_read_route(&self, _path: &str) {} - - /// Record read-route counters in one place so hot paths stay explicit. - fn record_read_route(&self, source: ReadRouteSource) { - match source { - ReadRouteSource::Backup => { - self.read_backup_hits.fetch_add(1, Ordering::Relaxed); - } - ReadRouteSource::Primary => { - self.read_primary_hits.fetch_add(1, Ordering::Relaxed); - } - ReadRouteSource::Redirect => { - self.read_redirect_hits.fetch_add(1, Ordering::Relaxed); - } - ReadRouteSource::Miss => { - self.read_misses.fetch_add(1, Ordering::Relaxed); - } - } - } - - /// Export read-route metrics for operational introspection. - pub(crate) fn read_route_metrics(&self) -> Value { - json!({ - "cache_hits": 0, - "backup_hits": self.read_backup_hits.load(Ordering::Relaxed), - "primary_hits": self.read_primary_hits.load(Ordering::Relaxed), - "redirect_hits": self.read_redirect_hits.load(Ordering::Relaxed), - "misses": self.read_misses.load(Ordering::Relaxed), - }) - } - - /// Stat the first reachable redirect target and return user-visible metadata. - pub(super) async fn redirect_file_info( - &self, - path: &str, - name: &str, - redirect_entry: &RedirectEntry, - ) -> FileInfo { - for target_name in &redirect_entry.targets { - if let Some(be) = self.backup_by_name(target_name) { - if let Ok(mut info) = be.backend.stat(path).await { - info.name = name.to_string(); - return info; - } - } - } - FileInfo::new_file(name.to_string(), 0, 0o644) - } - - /// Resolve the read backend for a path using the fallback chain. - pub(super) async fn resolve_read_backend(&self, path: &str) -> Option> { - let normalized = normalize_prefix_path(path); - let read_backups = self.read_backups_sorted(); - let backup_exists = futures::future::join_all(read_backups.iter().map(|backup| async { - ( - backup.name.clone(), - backup.backend.clone(), - backup.backend.exists(&normalized).await, - ) - })) - .await; - for (_name, backend, exists) in backup_exists { - if exists { - self.record_read_route(ReadRouteSource::Backup); - return Some(backend); - } - } - - if self.primary().backend.exists(&normalized).await { - self.record_read_route(ReadRouteSource::Primary); - return Some(self.primary().backend.clone()); - } - - if self.redirects.is_empty() { - self.record_read_route(ReadRouteSource::Miss); - return None; - } - - let dir = parent_dir(&normalized); - let name = file_name(&normalized).to_string(); - let ctx = current_required_ctx() - .or_else(|_| self.meta_store.ctx_resolver().resolve(&dir)) - .ok()?; - if let Ok(redirect_meta) = self.meta_store.get_redirect_meta(&dir, &ctx).await { - if let Some(entry) = redirect_meta.entries.get(&name) { - let redirect_targets: Vec<(String, Arc)> = entry - .targets - .iter() - .filter_map(|target_name| { - self.backup_by_name(target_name) - .map(|be| (be.name.clone(), be.backend.clone())) - }) - .collect(); - let redirect_exists = futures::future::join_all(redirect_targets.iter().map( - |(target_name, backend)| async { - ( - target_name.clone(), - backend.clone(), - backend.exists(&normalized).await, - ) - }, - )) - .await; - for (_target_name, backend, exists) in redirect_exists { - if exists { - self.record_read_route(ReadRouteSource::Redirect); - return Some(backend); - } - } - } - } - - self.record_read_route(ReadRouteSource::Miss); - None - } -} diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rs deleted file mode 100644 index 1a6752d88b..0000000000 --- a/crates/ragfs/src/core/multibackend_wrapper/tests.rs +++ /dev/null @@ -1,939 +0,0 @@ -use std::sync::Arc; - -use serde_json::Value; - -use super::*; -use crate::core::context::FsContextInner; -use crate::plugins::memfs::MemFileSystem; - -/// Create the shared test context used by multi-write tests. -fn test_ctx() -> FsContext { - Arc::new(FsContextInner::new("acct".to_string())) -} - -/// Create a sync multi-write filesystem with one memfs backup. -fn test_multiwrite_fs(redirects: Vec) -> MultiWriteWrappedFS { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup: Arc = Arc::new(MemFileSystem::new()); - let builder = MultiWriteWrappedFS::builder(primary.clone()) - .with_primary_raw_backend(primary) - .with_backups(vec![BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup, - raw_backend: None, - operations: Vec::new(), - excludes: Vec::new(), - }]) - .sync_mode(SyncMode::Sync { - ack_count: 1, - timeout_ms: 0, - }); - - if redirects.is_empty() { - builder - } else { - builder.with_redirects(redirects) - } - .build() - .unwrap() -} - -#[test] -fn test_file_policy_over_size() { - let policy = RedirectPolicy::FileOverSizePolicy { - max_size_mb: 1, - target: Some(vec!["backup1".to_string()]), - }; - assert!(policy.matches("/a/big.bin", 2 * 1024 * 1024)); - assert!(!policy.matches("/a/small.txt", 512)); -} - -#[test] -fn test_file_policy_extension() { - let policy = RedirectPolicy::FileExtensionPolicy { - extensions: vec!["(pdf|ppt)".to_string()], - target: Some(vec!["backup1".to_string()]), - }; - assert!(policy.matches("/a/doc.pdf", 0)); - assert!(policy.matches("/a/slides.ppt", 0)); - assert!(!policy.matches("/a/text.txt", 0)); -} - -#[tokio::test] -async fn test_read_dir_redirect_entries_use_target_stat() { - let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.pdf$".to_string()], - target: Some(vec!["backup1".to_string()]), - }]); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx, async { - fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755) - .await?; - fs.write( - "/local/acct/docs/report.pdf", - b"pdf body", - 0, - WriteFlag::Create, - ) - .await?; - - let entries = fs - .read_dir("/local/acct/docs", None, None, None, None) - .await?; - let report = entries - .iter() - .find(|entry| entry.name == "report.pdf") - .expect("redirected file should be visible in read_dir"); - assert_eq!(report.size, 8); - assert_eq!(report.mode, 0o644); - Ok::<(), Error>(()) - }) - .await - .unwrap(); - - let no_redirect_fs = test_multiwrite_fs(Vec::new()); - let no_redirect_ctx = test_ctx(); - - FS_CTX - .scope(no_redirect_ctx.clone(), async { - no_redirect_fs - .ensure_parent_dirs("/local/acct/docs/real.txt", 0o755) - .await?; - no_redirect_fs - .write( - "/local/acct/docs/real.txt", - b"real body", - 0, - WriteFlag::Create, - ) - .await?; - no_redirect_fs - .inner - .meta_store - .update_dir_meta( - "/local/acct/docs", - &no_redirect_ctx, - |redirect, _sync_log| { - redirect.entries.insert( - "ghost.pdf".to_string(), - RedirectEntry { - targets: vec!["backup1".to_string()], - }, - ); - Ok(()) - }, - ) - .await?; - let backup = no_redirect_fs - .inner - .backup_by_name("backup1") - .expect("backup exists") - .backend - .clone(); - backup - .ensure_parent_dirs("/local/acct/docs/ghost.pdf", 0o755) - .await?; - backup - .write( - "/local/acct/docs/ghost.pdf", - b"GHOST_MARKER\n", - 0, - WriteFlag::Create, - ) - .await?; - - let entries = no_redirect_fs - .read_dir("/local/acct/docs", None, None, None, None) - .await?; - let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect(); - assert_eq!(names, vec!["real.txt"]); - - let tree = no_redirect_fs - .tree_directory( - "/local/acct", - false, - None, - Some(3), - None, - None, - None, - false, - ) - .await?; - let paths: Vec<&str> = tree.iter().map(|entry| entry.path.as_str()).collect(); - assert!(!paths.contains(&"/local/acct/docs/ghost.pdf")); - - let grep = no_redirect_fs - .grep( - "/local/acct/docs", - "GHOST_MARKER", - GrepOptions { - level_limit: Some(1), - ..Default::default() - }, - ) - .await?; - assert_eq!(grep.count, 0); - - let err = no_redirect_fs - .rename("/local/acct/docs/ghost.pdf", "/local/acct/docs/renamed.pdf") - .await - .expect_err("stale redirect metadata is ignored when redirects are disabled"); - assert!(matches!(err, Error::NotFound(_))); - assert!(backup.exists("/local/acct/docs/ghost.pdf").await); - assert!(!backup.exists("/local/acct/docs/renamed.pdf").await); - - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_recursive_grep_finds_nested_redirected_files() { - let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.md$".to_string()], - target: Some(vec!["backup1".to_string()]), - }]); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx, async { - fs.ensure_parent_dirs("/local/acct/resources/doc-1/page.md", 0o755) - .await?; - fs.write( - "/local/acct/resources/doc-1/page.md", - b"MULTIWRITE_TEST_MARKER\n", - 0, - WriteFlag::Create, - ) - .await?; - - let shallow = fs - .tree_directory( - "/local/acct/resources", - false, - None, - Some(1), - None, - None, - None, - false, - ) - .await?; - let shallow_paths: Vec<&str> = - shallow.iter().map(|entry| entry.path.as_str()).collect(); - assert_eq!(shallow_paths, vec!["/local/acct/resources/doc-1"]); - - let root_only = fs - .tree_directory( - "/local/acct/resources", - false, - None, - Some(0), - None, - None, - None, - false, - ) - .await?; - assert!(root_only.is_empty()); - - let deep = fs - .tree_directory( - "/local/acct/resources", - false, - None, - Some(2), - None, - None, - None, - false, - ) - .await?; - let deep_paths: Vec<&str> = deep.iter().map(|entry| entry.path.as_str()).collect(); - assert_eq!( - deep_paths, - vec![ - "/local/acct/resources/doc-1", - "/local/acct/resources/doc-1/page.md", - ] - ); - - let capped = fs - .tree_directory( - "/local/acct/resources", - false, - Some(1), - Some(2), - None, - None, - None, - false, - ) - .await?; - let capped_paths: Vec<&str> = capped.iter().map(|entry| entry.path.as_str()).collect(); - assert_eq!(capped_paths, vec!["/local/acct/resources/doc-1"]); - - let result = fs - .grep( - "/local/acct/resources", - "MULTIWRITE_TEST_MARKER", - GrepOptions { - recursive: true, - level_limit: Some(5), - ..Default::default() - }, - ) - .await?; - assert_eq!(result.count, 1); - assert_eq!(result.matches.len(), 1); - assert_eq!(result.matches[0].file, "doc-1/page.md"); - assert_eq!(result.matches[0].line, 1); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_redirect_write_marks_first_target_acked_before_returning() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup1: Arc = Arc::new(MemFileSystem::new()); - let backup2: Arc = Arc::new(MemFileSystem::new()); - let backup1_handle = backup1.clone(); - let fs = MultiWriteWrappedFS::builder(primary.clone()) - .with_primary_raw_backend(primary) - .with_backups(vec![ - BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup1, - raw_backend: None, - operations: Vec::new(), - excludes: Vec::new(), - }, - BackendEntry { - name: "backup2".to_string(), - role: BackendRole::Backup, - backend: backup2, - raw_backend: None, - operations: Vec::new(), - excludes: Vec::new(), - }, - ]) - .with_redirects(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.pdf$".to_string()], - target: Some(vec!["backup1".to_string(), "backup2".to_string()]), - }]) - .sync_mode(SyncMode::Async) - .build() - .unwrap(); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755) - .await?; - fs.write( - "/local/acct/docs/report.pdf", - b"pdf body", - 0, - WriteFlag::Create, - ) - .await?; - - let redirect_meta = fs - .inner - .meta_store - .get_redirect_meta("/local/acct/docs", &ctx) - .await?; - let entry = redirect_meta - .entries - .get("report.pdf") - .expect("redirect metadata should only appear after one target succeeds"); - assert_eq!( - entry.targets, - vec!["backup1".to_string(), "backup2".to_string()] - ); - - let sync_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/docs", &ctx) - .await?; - let log_entry = sync_log - .entries - .get("report.pdf") - .expect("redirected file should be tracked in sync log"); - assert!(log_entry.is_primary_committed()); - assert!( - log_entry.is_in_sync("backup1"), - "the first redirect target must be acknowledged before returning" - ); - assert!( - backup1_handle.exists("/local/acct/docs/report.pdf").await, - "the first redirect target must already contain the file" - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_redirect_write_does_not_publish_metadata_when_first_target_fails() { - let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.pdf$".to_string()], - target: Some(vec!["missing-backup".to_string()]), - }]); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - let err = fs - .write( - "/local/acct/docs/fail.pdf", - b"content", - 0, - WriteFlag::Create, - ) - .await - .expect_err("redirect write should fail when the first target is not durable"); - - assert!( - err.to_string().contains("resolved no writable targets"), - "unexpected error: {err}" - ); - - let redirect_meta = fs - .inner - .meta_store - .get_redirect_meta("/local/acct/docs", &ctx) - .await?; - assert!( - !redirect_meta.entries.contains_key("fail.pdf"), - "redirect metadata must not be published before the first target succeeds" - ); - - let sync_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/docs", &ctx) - .await?; - assert!( - !sync_log.entries.contains_key("fail.pdf"), - "sync log must not record a committed redirect write before durability" - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_cross_dir_rename_does_not_create_target_write_log() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/src/file.txt", 0o755) - .await?; - fs.ensure_parent_dirs("/local/acct/dst/file.txt", 0o755) - .await?; - fs.write( - "/local/acct/src/file.txt", - b"rename me", - 0, - WriteFlag::Create, - ) - .await?; - fs.rename("/local/acct/src/file.txt", "/local/acct/dst/file.txt") - .await?; - - let target_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/dst", &ctx) - .await?; - assert!( - !target_log.entries.contains_key("file.txt"), - "target directory must not record a fake write for cross-dir rename" - ); - let source_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/src", &ctx) - .await?; - let rename_entry = source_log - .entries - .get("file.txt") - .expect("source directory should record rename command"); - assert!(matches!( - &rename_entry.op, - SyncOp::Rename { to } if to == "/local/acct/dst/file.txt" - )); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_rename_redirected_file_renames_original_redirect_target() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup: Arc = Arc::new(MemFileSystem::new()); - let backup_handle = backup.clone(); - let fs = MultiWriteWrappedFS::builder(primary.clone()) - .with_primary_raw_backend(primary) - .with_backups(vec![BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup, - raw_backend: None, - operations: vec![OperationItemConfig { - operation: "read".to_string(), - priority: 1, - }], - excludes: Vec::new(), - }]) - .with_redirects(vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.pdf$".to_string()], - target: Some(vec!["backup1".to_string()]), - }]) - .sync_mode(SyncMode::Sync { - ack_count: 1, - timeout_ms: 0, - }) - .build() - .unwrap(); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx, async { - fs.ensure_parent_dirs("/local/acct/src/a.pdf", 0o755) - .await?; - fs.ensure_parent_dirs("/local/acct/dst/b.pdf", 0o755) - .await?; - fs.write( - "/local/acct/src/a.pdf", - b"pdf payload", - 0, - WriteFlag::Create, - ) - .await?; - assert!(backup_handle.exists("/local/acct/src/a.pdf").await); - - fs.rename("/local/acct/src/a.pdf", "/local/acct/dst/b.pdf") - .await?; - - assert!( - backup_handle.exists("/local/acct/dst/b.pdf").await, - "redirect target backend should receive rename fanout" - ); - assert!( - !backup_handle.exists("/local/acct/src/a.pdf").await, - "redirect target backend should not retain the old path" - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_copy_within_primary_preserves_primary_bytes_and_fanouts_backups() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup: Arc = Arc::new(MemFileSystem::new()); - let backup_handle = backup.clone(); - let fs = MultiWriteWrappedFS::builder(primary.clone()) - .with_primary_raw_backend(primary) - .with_backups(vec![BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup, - raw_backend: None, - operations: Vec::new(), - excludes: Vec::new(), - }]) - .sync_mode(SyncMode::Sync { - ack_count: 1, - timeout_ms: 0, - }) - .build() - .unwrap(); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/src.bin", 0o755) - .await?; - fs.write("/local/acct/docs/src.bin", b"payload", 0, WriteFlag::Create) - .await?; - - let copied = fs - .copy_within_primary("/local/acct/docs/src.bin", "/local/acct/docs/dst.bin") - .await?; - assert!( - copied, - "copy_within_primary should use the primary fast-path" - ); - - assert_eq!( - fs.inner - .primary() - .backend - .read("/local/acct/docs/dst.bin", 0, 0) - .await?, - b"payload" - ); - assert_eq!( - backup_handle.read("/local/acct/docs/dst.bin", 0, 0).await?, - b"payload" - ); - - let sync_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/docs", &ctx) - .await?; - let entry = sync_log - .entries - .get("dst.bin") - .expect("copied file should be tracked for backup fanout"); - assert!(entry.is_primary_committed()); - assert!(matches!(entry.op, SyncOp::SyncFile { size } if size == 7)); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_sync_log_write_entry_does_not_embed_large_payload() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - let payload = vec![b'x'; 256 * 1024]; - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755) - .await?; - fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create) - .await?; - - let sync_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/docs", &ctx) - .await?; - let encoded = serde_json::to_vec(&sync_log)?; - assert!( - encoded.len() < 16 * 1024, - "sync log should stay metadata-sized, got {} bytes", - encoded.len() - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_sync_log_truncate_entry_does_not_embed_snapshot() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - let payload = vec![b'y'; 256 * 1024]; - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755) - .await?; - fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create) - .await?; - fs.truncate("/local/acct/docs/large.bin", 128).await?; - - let sync_log = fs - .inner - .meta_store - .get_sync_log_meta("/local/acct/docs", &ctx) - .await?; - let encoded = serde_json::to_vec(&sync_log)?; - assert!( - encoded.len() < 16 * 1024, - "truncate sync log should stay metadata-sized, got {} bytes", - encoded.len() - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_fanout_sync_returns_structured_quorum_error() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - let targets = fs.inner.write_targets("/local/acct/docs/fail.txt", 0); - - let err = Inner::fanout_sync( - &fs.inner, - "/local/acct/docs/fail.txt", - &targets, - &ctx, - BackupWriteOp::Replay(SyncOp::SyncFile { size: 1 }), - ) - .await - .expect_err("sync fanout should surface structured quorum failure"); - - match err { - Error::SyncWriteQuorum { - succeeded, - required, - attempted, - failures, - } => { - assert_eq!(succeeded, 0); - assert_eq!(required, 1); - assert_eq!(attempted, 1); - assert_eq!(failures.len(), 1); - assert_eq!(failures[0].backend, "backup1"); - assert_eq!(failures[0].kind, "not_found"); - assert_eq!(failures[0].message, "not found: /local/acct/docs/fail.txt"); - } - other => panic!("unexpected error: {other}"), - } -} - -#[tokio::test] -async fn test_sync_write_errors_when_required_ack_has_no_targets() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup: Arc = Arc::new(MemFileSystem::new()); - let fs = MultiWriteWrappedFS::builder(primary.clone()) - .with_primary_raw_backend(primary) - .with_backups(vec![BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup, - raw_backend: None, - operations: Vec::new(), - excludes: vec![RedirectPolicy::FileExtensionPolicy { - extensions: vec!["\\.txt$".to_string()], - target: None, - }], - }]) - .sync_mode(SyncMode::Sync { - ack_count: 1, - timeout_ms: 0, - }) - .build() - .unwrap(); - let ctx = test_ctx(); - - let err = FS_CTX - .scope(ctx, async { - fs.ensure_parent_dirs("/local/acct/docs/no-target.txt", 0o755) - .await?; - fs.write( - "/local/acct/docs/no-target.txt", - b"payload", - 0, - WriteFlag::Create, - ) - .await - }) - .await - .expect_err("sync mode must not silently pass when all backups are excluded"); - - match err { - Error::SyncWriteQuorum { - succeeded, - required, - attempted, - failures, - } => { - assert_eq!(succeeded, 0); - assert_eq!(required, 1); - assert_eq!(attempted, 0); - assert!(failures.is_empty()); - } - other => panic!("unexpected error: {other}"), - } -} - -#[tokio::test] -async fn test_read_route_metrics_capture_backup_fallbacks() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let backup: Arc = Arc::new(MemFileSystem::new()); - let backup_handle = backup.clone(); - let fs = MultiWriteWrappedFS::builder(primary) - .with_backups(vec![BackendEntry { - name: "backup1".to_string(), - role: BackendRole::Backup, - backend: backup, - raw_backend: None, - operations: vec![OperationItemConfig { - operation: "read".to_string(), - priority: 1, - }], - excludes: Vec::new(), - }]) - .build() - .unwrap(); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx, async { - backup_handle - .ensure_parent_dirs("/local/acct/hot/cache.txt", 0o755) - .await?; - backup_handle - .write("/local/acct/hot/cache.txt", b"hot", 0, WriteFlag::Create) - .await?; - - assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot"); - assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot"); - - let metrics = fs.inner.read_route_metrics(); - assert_eq!(metrics.get("backup_hits").and_then(Value::as_u64), Some(2)); - assert_eq!(metrics.get("cache_hits").and_then(Value::as_u64), Some(0)); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_shutdown_drains_background_retry_loop() { - let fs = test_multiwrite_fs(Vec::new()); - fs.shutdown().await.unwrap(); -} - -#[tokio::test] -async fn test_system_sync_status_reports_corrupt_redirect_meta() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755) - .await?; - fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create) - .await?; - fs.inner - .primary() - .backend - .write( - "/local/acct/docs/.redirect.json", - b"{not-json", - 0, - WriteFlag::Create, - ) - .await?; - - let err = fs - .system_sync_status("/local/acct/docs/a.txt") - .await - .expect_err("corrupt redirect metadata must be reported"); - assert!( - err.to_string().contains("redirect") - || err.to_string().contains("json") - || err.to_string().contains("serialization"), - "unexpected error: {err}" - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_retry_pending_dir_reports_corrupt_redirect_meta() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755) - .await?; - fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create) - .await?; - fs.inner - .meta_store - .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| { - sync_log - .entries - .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create)); - Ok(()) - }) - .await?; - fs.inner - .primary() - .backend - .write( - "/local/acct/docs/.redirect.json", - b"{not-json", - 0, - WriteFlag::Create, - ) - .await?; - - let err = fs - .inner - .retry_pending_dir("/local/acct/docs") - .await - .expect_err("corrupt redirect metadata must be reported to retry logic"); - assert!( - err.to_string().contains("redirect") - || err.to_string().contains("json") - || err.to_string().contains("serialization"), - "unexpected error: {err}" - ); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn test_uncommitted_sync_log_entry_is_visible_but_not_retried() { - let fs = test_multiwrite_fs(Vec::new()); - let ctx = test_ctx(); - - FS_CTX - .scope(ctx.clone(), async { - fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755) - .await?; - fs.inner - .meta_store - .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| { - sync_log - .entries - .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create)); - Ok(()) - }) - .await?; - - let status = fs.system_sync_status("/local/acct/docs").await?; - assert_eq!(status["entry_count"], 1); - assert_eq!(status["pending_target_count"], 0); - assert_eq!(status["entries"][0]["primary_committed"], false); - assert_eq!( - status["entries"][0]["targets"][0]["primary_committed"], - false - ); - - let retry = fs.system_sync_retry("/local/acct/docs").await?; - assert_eq!(retry["retried"], 0); - assert_eq!(retry["failed"], 0); - assert_eq!(retry["skipped"], 1); - assert_eq!(retry["results"][0]["status"], "awaiting_primary_commit"); - Ok::<(), Error>(()) - }) - .await - .unwrap(); -} diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs index fff95a76a5..9038d4abf3 100644 --- a/crates/ragfs/src/core/types.rs +++ b/crates/ragfs/src/core/types.rs @@ -397,10 +397,6 @@ pub struct PluginConfig { /// Primary encryption enabled (follows global, not independently configurable) #[serde(default)] pub primary_encryption_enabled: bool, - - /// Primary redirect policies - #[serde(default)] - pub primary_redirects: Vec, } impl Default for PluginConfig { @@ -413,7 +409,6 @@ impl Default for PluginConfig { backups: None, server_encryption_enabled: false, primary_encryption_enabled: false, - primary_redirects: Vec::new(), } } } @@ -444,8 +439,6 @@ impl PluginConfig { Self::take_bool_with_default("server_encryption_enabled", &mut params, false)?; let primary_encryption_enabled = Self::take_bool_with_default("primary_encryption_enabled", &mut params, false)?; - let primary_redirects = - Self::take_json_with_default("primary_redirects", &mut params, Vec::new())?; Ok(Self { name: name.into(), @@ -454,7 +447,6 @@ impl PluginConfig { backups, server_encryption_enabled, primary_encryption_enabled, - primary_redirects, }) } @@ -496,15 +488,6 @@ impl PluginConfig { } } - /// Remove one optional JSON config value and fall back to the provided default. - fn take_json_with_default( - field_name: &str, - params: &mut HashMap, - default: T, - ) -> crate::core::Result { - Ok(Self::take_optional_json(field_name, params)?.unwrap_or(default)) - } - /// Return a human-readable config value kind for error messages. fn config_value_kind(value: &ConfigValue) -> &'static str { match value { @@ -576,32 +559,41 @@ impl ConfigValue { /// Multi-write backends container configuration #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BackendsConfig { - /// Sync type: "sync" or "async", default async - #[serde(default = "default_sync_type")] - pub sync_type: String, - /// Minimum backup ack count for sync mode - pub write_ack_count: Option, - /// Timeout for waiting backup ack in sync mode (ms) - pub write_ack_timeout_ms: Option, - /// Max concurrent async writes - pub write_concurrency: Option, - /// Retry loop interval in milliseconds - pub retry_interval_ms: Option, - /// Base backoff in milliseconds for retry attempts - pub retry_backoff_base_ms: Option, - /// Maximum retry attempts per file/target in one round - pub retry_max_retries_per_round: Option, - /// Failure threshold before quarantining one file/target pair - pub retry_quarantine_after_failures: Option, - /// Deprecated no-op retained for config compatibility. - #[serde(default)] - pub read_probe_cache_ttl_ms: Option, + /// Cache key namespace used by cache-backed metadata. + #[serde(default = "default_backups_namespace")] + pub namespace: String, + /// Initial number of metadata partitions. + #[serde(default = "default_initial_partitions")] + pub initial_partitions: u32, + /// Checkpoint interval in seconds. + #[serde(default = "default_checkpoint_interval_secs")] + pub checkpoint_interval_secs: u64, + /// Metadata storage provider. + #[serde(default = "default_backups_provider")] + pub provider: String, /// Backup items + #[serde(default)] pub items: Vec, } -fn default_sync_type() -> String { - "async".to_string() +/// Return the default cache-backed metadata namespace. +fn default_backups_namespace() -> String { + "default".to_string() +} + +/// Return the default metadata partition count. +fn default_initial_partitions() -> u32 { + 16 +} + +/// Return the default checkpoint interval in seconds. +fn default_checkpoint_interval_secs() -> u64 { + 86_400 +} + +/// Return the default metadata provider. +fn default_backups_provider() -> String { + "filesystem".to_string() } /// Single backup backend item configuration @@ -614,23 +606,8 @@ pub struct BackendItemConfig { /// Plugin-specific params (nested JSON) #[serde(default)] pub params: serde_json::Value, - /// Timeout in seconds - pub timeout: Option, /// Encryption config for this backup pub encryption: Option, - /// Operations this backup participates in - pub operations: Option>, - /// Exclude policies - pub excludes: Option>, -} - -/// Per-operation priority config -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OperationItemConfig { - /// Operation type: "read" | "write" - pub operation: String, - /// Priority (smaller = higher priority) - pub priority: u32, } /// Encryption on/off config for a backend @@ -640,211 +617,6 @@ pub struct EncryptionConfig { pub enabled: bool, } -/// Redirect / exclude policy (shared trait) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum RedirectPolicy { - /// Redirect/exclude files exceeding a size threshold - #[serde(rename = "FileOverSizePolicy")] - FileOverSizePolicy { - /// Max file size in MB - max_size_mb: u64, - /// Target backend names (redirect only) - target: Option>, - }, - /// Redirect/exclude files matching extension patterns - #[serde(rename = "FileExtensionPolicy")] - FileExtensionPolicy { - /// Regex patterns for file extensions - extensions: Vec, - /// Target backend names (redirect only) - target: Option>, - }, -} - -/// Strongly typed multi-write operation stored in `.sync_log.json`. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "type")] -pub enum SyncOp { - /// Create an empty file. - Create, - /// Synchronize file content by copying the current primary state to the backup. - SyncFile { - /// File size used by retry/exclude policy decisions. - size: u64, - }, - /// Create a directory. - Mkdir { - /// Directory mode. - mode: u32, - }, - /// Change file mode. - Chmod { - /// File mode. - mode: u32, - }, - /// Remove one file. - Remove, - /// Remove a file tree. - RemoveAll, - /// Rename to another path. - Rename { - /// Rename target path. - to: String, - }, -} - -/// Per-backend sync state -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] -pub struct BackendSyncState { - /// Acknowledged sequence number - pub acked_seq: u64, - /// Consecutive retry failures for the current operation. - #[serde(default)] - pub retry_failures: u32, - /// Whether this backend/path pair is quarantined for manual intervention. - #[serde(default)] - pub quarantined: bool, -} - -impl BackendSyncState { - /// Create an acknowledged backend sync state. - pub fn acked(acked_seq: u64) -> Self { - Self { - acked_seq, - retry_failures: 0, - quarantined: false, - } - } - - /// Mark this backend as acknowledged and clear retry state. - pub fn mark_acked(&mut self, acked_seq: u64) { - self.acked_seq = acked_seq; - self.retry_failures = 0; - self.quarantined = false; - } - - /// Record one retry failure and quarantine after the configured threshold. - pub fn mark_retry_failed(&mut self, quarantine_after_failures: u32) { - self.retry_failures = self.retry_failures.saturating_add(1); - if self.retry_failures >= quarantine_after_failures { - self.quarantined = true; - } - } -} - -/// Sync log entry for a single file path. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct SyncLogEntry { - /// Latest monotonic sequence number. - pub latest_seq: u64, - /// Whether the primary backend has durably completed this operation. - #[serde(default = "default_primary_committed")] - pub primary_committed: bool, - /// Strongly typed operation payload. - pub op: SyncOp, - /// Per-backend acked sequence numbers. - pub backends: std::collections::HashMap, -} - -fn default_primary_committed() -> bool { - true -} - -impl SyncLogEntry { - /// Create a new prepared sync log entry for a sequenced operation. - pub fn new(latest_seq: u64, op: SyncOp) -> Self { - Self { - latest_seq, - primary_committed: false, - op, - backends: std::collections::HashMap::new(), - } - } - - /// Create a committed sync log entry for an operation already applied on primary. - pub fn committed(latest_seq: u64, op: SyncOp) -> Self { - Self { - latest_seq, - primary_committed: true, - op, - backends: std::collections::HashMap::new(), - } - } - - /// Mark this entry as committed on primary. - pub fn mark_primary_committed(&mut self) { - self.primary_committed = true; - } - - /// Return whether this entry is committed on primary. - pub fn is_primary_committed(&self) -> bool { - self.primary_committed - } - - /// Return the backend sync state if present. - pub fn backend_state(&self, backend_name: &str) -> Option<&BackendSyncState> { - self.backends.get(backend_name) - } - - /// Return the acknowledged sequence for a backend, or 0 if unknown. - pub fn acked_seq(&self, backend_name: &str) -> u64 { - self.backend_state(backend_name) - .map(|state| state.acked_seq) - .unwrap_or(0) - } - - /// Return whether the backend is quarantined. - pub fn is_quarantined(&self, backend_name: &str) -> bool { - self.backend_state(backend_name) - .map(|state| state.quarantined) - .unwrap_or(false) - } - - /// Return whether the backend has acknowledged the latest operation. - pub fn is_in_sync(&self, backend_name: &str) -> bool { - self.acked_seq(backend_name) >= self.latest_seq - } -} - -/// Sync log file content -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] -pub struct SyncLogMeta { - /// File entries keyed by file name (current directory) - pub entries: std::collections::HashMap, -} - -/// Redirect metadata file content -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RedirectMeta { - /// Schema version - #[serde(default = "default_redirect_version")] - pub version: u32, - /// Redirect entries keyed by file name - #[serde(default)] - pub entries: std::collections::HashMap, -} - -fn default_redirect_version() -> u32 { - 1 -} - -impl Default for RedirectMeta { - fn default() -> Self { - Self { - version: 1, - entries: HashMap::new(), - } - } -} - -/// Single redirect entry -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RedirectEntry { - /// Target backend names - pub targets: Vec, -} - /// Backend role #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendRole { @@ -854,15 +626,6 @@ pub enum BackendRole { Backup, } -/// Sync type enum -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SyncType { - /// Synchronous: wait for backup ack before returning - Sync, - /// Asynchronous: return after primary write, sync in background - Async, -} - /// Custom serde module for SystemTime mod systemtime_serde { use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -922,21 +685,6 @@ mod tests { assert_eq!(param.param_type, "string"); } - #[test] - fn test_sync_log_entry_legacy_payload_defaults_primary_committed_true() { - let entry: SyncLogEntry = serde_json::from_value(serde_json::json!({ - "latest_seq": 7, - "op": { - "type": "Create" - }, - "backends": {} - })) - .unwrap(); - - assert!(entry.is_primary_committed()); - assert_eq!(entry.latest_seq, 7); - } - #[test] fn test_plugin_config_from_raw_parts_handles_multiwrite_shapes() { let mut valid_params = HashMap::new(); @@ -958,22 +706,10 @@ mod tests { "primary_encryption_enabled".to_string(), ConfigValue::Bool(true), ); - valid_params.insert( - "primary_redirects".to_string(), - ConfigValue::Json(serde_json::json!([ - { - "type": "FileExtensionPolicy", - "extensions": [".bin"], - "target": ["backup1"] - } - ])), - ); - let config = PluginConfig::from_raw_parts("localfs", "/local", valid_params).unwrap(); assert!(config.backups.is_some()); assert!(config.server_encryption_enabled); assert!(config.primary_encryption_enabled); - assert_eq!(config.primary_redirects.len(), 1); assert_eq!( config.params.get("root_path"), Some(&ConfigValue::String("/tmp/data".to_string())) @@ -981,7 +717,6 @@ mod tests { assert!(!config.params.contains_key("backups")); assert!(!config.params.contains_key("server_encryption_enabled")); assert!(!config.params.contains_key("primary_encryption_enabled")); - assert!(!config.params.contains_key("primary_redirects")); for (params, expected_message) in [ ( diff --git a/crates/ragfs/src/git/service.rs b/crates/ragfs/src/git/service.rs index 896e7941bc..0f1548ae62 100644 --- a/crates/ragfs/src/git/service.rs +++ b/crates/ragfs/src/git/service.rs @@ -6635,7 +6635,9 @@ mod fast_path1_tests { ) .unwrap(); - let stack = build_default_stack(RagfsConfig::default()).await.unwrap(); + let mut ragfs_config = RagfsConfig::default(); + ragfs_config.pathlock.provider = "memory".to_string(); + let stack = build_default_stack(ragfs_config).await.unwrap(); let mut params = HashMap::new(); params.insert( "local_dir".to_string(), @@ -6648,25 +6650,17 @@ mod fast_path1_tests { mount_path: "/local".to_string(), params, backups: Some(BackendsConfig { - sync_type: "async".to_string(), - write_ack_count: None, - write_ack_timeout_ms: None, - write_concurrency: None, - retry_interval_ms: None, - retry_backoff_base_ms: None, - retry_max_retries_per_round: None, - retry_quarantine_after_failures: None, - read_probe_cache_ttl_ms: None, + namespace: "default".to_string(), + initial_partitions: 16, + checkpoint_interval_secs: 86_400, + provider: "filesystem".to_string(), items: vec![BackendItemConfig { name: "backup1".to_string(), backend: "localfs".to_string(), params: serde_json::json!({ "local_dir": backup_dir.path().to_str().unwrap(), }), - timeout: None, encryption: None, - operations: None, - excludes: None, }], }), ..PluginConfig::default() diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs index 668263a610..cef2bdb7e8 100644 --- a/crates/ragfs/src/lock/manager.rs +++ b/crates/ragfs/src/lock/manager.rs @@ -875,7 +875,7 @@ impl PathLockManager { } if waiting.0.is_none() { - info!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state"); + debug!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state"); waiting.0 = Some(&self.waiting_lock_count); self.waiting_lock_count .fetch_add(1, AtomicOrdering::Relaxed); diff --git a/crates/ragfs/src/multibackend/admin.rs b/crates/ragfs/src/multibackend/admin.rs deleted file mode 100644 index d0450a1b3b..0000000000 --- a/crates/ragfs/src/multibackend/admin.rs +++ /dev/null @@ -1,234 +0,0 @@ -use std::collections::HashSet; -use std::time::Duration; - -use serde_json::{json, Value}; - -use crate::core::errors::Result; -use crate::core::filesystem::{normalize_prefix_path, FileSystem}; -use crate::core::multibackend_wrapper::{MultiWriteWrappedFS, SyncWorkEntry}; -use crate::multibackend::meta::{current_required_ctx, parent_dir}; - -impl MultiWriteWrappedFS { - /// Collect effective sync work entries under a path using the current request context. - async fn collect_sync_work(&self, path: &str) -> Result> { - let ctx = current_required_ctx()?; - let inner = &self.inner; - let normalized = normalize_prefix_path(path); - let path_info = ::stat(self, &normalized).await?; - let mut dirs = Vec::new(); - let mut seen_dirs = HashSet::new(); - - let add_dir = |dirs: &mut Vec, seen_dirs: &mut HashSet, dir: String| { - if seen_dirs.insert(dir.clone()) { - dirs.push(dir); - } - }; - - if path_info.is_dir { - add_dir(&mut dirs, &mut seen_dirs, normalized.clone()); - for dir in inner.pending_dirs_snapshot().await { - if dir == normalized - || dir - .strip_prefix(&normalized) - .is_some_and(|suffix| suffix.starts_with('/')) - { - add_dir(&mut dirs, &mut seen_dirs, dir); - } - } - if dirs.len() == 1 { - for entry in inner - .primary() - .backend - .tree_directory(&normalized, true, None, None, None, None, None, false) - .await? - { - if entry.info.is_dir { - add_dir( - &mut dirs, - &mut seen_dirs, - normalize_prefix_path(&entry.path), - ); - } - } - } - } else { - add_dir( - &mut dirs, - &mut seen_dirs, - normalize_prefix_path(&parent_dir(&normalized)), - ); - } - - let mut work = Vec::new(); - for dir in dirs { - work.extend( - inner - .sync_work_in_dir(&dir, &ctx) - .await? - .into_iter() - .filter(|work| path_info.is_dir || work.file_path == normalized), - ); - } - - Ok(work) - } - - /// Query effective multi-write sync status under a file or directory path. - pub async fn system_sync_status(&self, path: &str) -> Result { - let work = self.collect_sync_work(path).await?; - let mut entries = Vec::new(); - let mut pending_target_count = 0usize; - - for work in work { - let mut targets = Vec::new(); - let primary_committed = work.entry.is_primary_committed(); - let mut all_synced = primary_committed; - - for backend_name in work.targets { - let acked_seq = if primary_committed { - work.entry.acked_seq(&backend_name) - } else { - 0 - }; - let in_sync = primary_committed && work.entry.is_in_sync(&backend_name); - if primary_committed && !in_sync { - pending_target_count += 1; - all_synced = false; - } - let state = work.entry.backend_state(&backend_name); - targets.push(json!({ - "name": backend_name, - "acked_seq": acked_seq, - "retry_failures": state.map(|state| state.retry_failures).unwrap_or(0), - "quarantined": state.map(|state| state.quarantined).unwrap_or(false), - "primary_committed": primary_committed, - "in_sync": in_sync, - })); - } - - entries.push(json!({ - "path": work.file_path, - "latest_seq": work.entry.latest_seq, - "primary_committed": primary_committed, - "op": serde_json::to_value(&work.entry.op)?, - "all_synced": all_synced, - "targets": targets, - })); - } - - entries.sort_by(|a, b| { - let ap = a.get("path").and_then(Value::as_str).unwrap_or_default(); - let bp = b.get("path").and_then(Value::as_str).unwrap_or_default(); - ap.cmp(bp) - }); - - Ok(json!({ - "path": normalize_prefix_path(path), - "entry_count": entries.len(), - "pending_target_count": pending_target_count, - "capabilities": { - "multi_instance_safe": false - }, - "read_route_metrics": self.inner.read_route_metrics(), - "entries": entries, - })) - } - - /// Manually retry lagging multi-write targets under a file or directory path. - pub async fn system_sync_retry(&self, path: &str) -> Result { - let ctx = current_required_ctx()?; - let work = self.collect_sync_work(path).await?; - let mut results = Vec::new(); - let mut retried = 0usize; - let mut failed = 0usize; - let mut skipped = 0usize; - - for work in work { - if !work.entry.is_primary_committed() { - skipped += work.targets.len(); - for backend_name in work.targets { - results.push(json!({ - "path": work.file_path, - "target": backend_name, - "status": "awaiting_primary_commit", - "latest_seq": work.entry.latest_seq, - "primary_committed": false, - "acked_seq": 0, - })); - } - continue; - } - - for backend_name in work.targets { - let acked_seq = work.entry.acked_seq(&backend_name); - let was_quarantined = work.entry.is_quarantined(&backend_name); - if work.entry.is_in_sync(&backend_name) { - skipped += 1; - results.push(json!({ - "path": work.file_path, - "target": backend_name, - "status": "skipped", - "latest_seq": work.entry.latest_seq, - "acked_seq": acked_seq, - })); - continue; - } - - let mut last_error = None; - let mut success = false; - for _attempt in 0..self.inner.max_retry_per_round { - match self - .inner - .replay_operation(&work.file_path, &backend_name, &ctx) - .await - { - Ok(()) => { - success = true; - break; - } - Err(err) => { - last_error = Some(err.to_string()); - tokio::time::sleep(Duration::from_millis( - self.inner.retry_backoff_base_ms, - )) - .await; - } - } - } - - if success { - retried += 1; - results.push(json!({ - "path": work.file_path, - "target": backend_name, - "status": "retried", - "latest_seq": work.entry.latest_seq, - "acked_seq": work.entry.latest_seq, - })); - } else { - self.inner - .record_backup_retry_failure(&work.file_path, &backend_name, &ctx) - .await?; - failed += 1; - results.push(json!({ - "path": work.file_path, - "target": backend_name, - "status": "failed", - "latest_seq": work.entry.latest_seq, - "acked_seq": acked_seq, - "was_quarantined": was_quarantined, - "error": last_error.unwrap_or_else(|| "unknown replay error".to_string()), - })); - } - } - } - - Ok(json!({ - "path": normalize_prefix_path(path), - "retried": retried, - "failed": failed, - "skipped": skipped, - "results": results, - })) - } -} diff --git a/crates/ragfs/src/multibackend/catch_up.rs b/crates/ragfs/src/multibackend/catch_up.rs new file mode 100644 index 0000000000..0ed190b74a --- /dev/null +++ b/crates/ragfs/src/multibackend/catch_up.rs @@ -0,0 +1,1058 @@ +//! Backup catch-up workers and current-state reconciliation primitives. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::future::Future; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use rand::Rng; +use serde_json::Value; +use tokio::sync::watch; +use tracing::{debug, warn}; + +use crate::core::context::{FsContext, FsContextInner, FS_CTX}; +use crate::core::errors::{Error, Result}; +use crate::core::filesystem::FileSystem; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::core::types::{FileInfo, ListSortBy, SortOrder, WriteFlag}; +use crate::lock::OwnedPathLockLease; +use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR}; +use crate::multibackend::meta::{MetadataStore, MultiWriteWorker}; +use crate::multibackend::model::{ + FileState, PartitionState, PartitionsManifest, ScopeKey, SegmentEventType, SegmentRecord, +}; +use crate::multibackend::provider::MultiWriteProvider; + +#[cfg(test)] +#[path = "tests/test_catch_up.rs"] +mod tests; + +/// One configured backup handle owned by its catch-up worker. +pub struct CatchUpTarget { + /// Stable backend id stored in partition progress. + pub backend_id: String, + /// Encrypted business filesystem handle. + pub backend: Arc, +} + +/// Parsed checkpoint state accepted by CatchUpWorker from the future reader. +#[derive(Clone)] +pub struct CheckpointSnapshot { + /// Last sequence represented by this complete checkpoint. + pub checkpoint_to_seq: u64, + /// Materialized file states already decoded by the checkpoint reader. + pub file_states: Vec, + /// Directory prefixes requiring current-state reconciliation. + pub directory_prefixes: Vec, +} + +/// Typed outcome of reading the latest checkpoint publication. +pub enum CheckpointReadResult { + /// No published checkpoint fits the requested sequence window. + NoCheckpoint, + /// One complete validated checkpoint snapshot. + Snapshot(CheckpointSnapshot), + /// Immutable files changed during a reader/GC race; reread latest. + RetryLatestCheckpoint, +} + +/// Consumer boundary implemented by the future checkpoint reader. +#[async_trait] +pub trait CheckpointConsumer: Send + Sync { + /// Return the latest usable parsed checkpoint inside the fixed tail. + async fn read_latest( + &self, + scope: &ScopeKey, + synced_seq: u64, + tail_seq: u64, + ) -> Result; +} + +struct EmptyCheckpointConsumer; + +#[async_trait] +impl CheckpointConsumer for EmptyCheckpointConsumer { + /// Return no checkpoint until T-011 supplies a reader. + async fn read_latest( + &self, + _scope: &ScopeKey, + _synced_seq: u64, + _tail_seq: u64, + ) -> Result { + Ok(CheckpointReadResult::NoCheckpoint) + } +} + +/// One backup worker that consumes V2 metadata without blocking primary writes. +pub struct CatchUpWorker { + backend_id: String, + primary: Arc, + backup: Arc, + store: Arc, + provider: Arc, + checkpoint_consumer: Arc, + cancellation: Option>, +} + +/// One Exact PathLock lease protecting a backup catch-up worker. +pub struct CatchUpLease { + store: Arc, + lease: OwnedPathLockLease, + valid: AtomicBool, +} + +enum CatchUpFailure { + Cancelled, + Failed(Error), +} + +impl From for CatchUpFailure { + /// Preserve ordinary failures while propagating explicit cancellation separately. + fn from(error: Error) -> Self { + Self::Failed(error) + } +} + +type CatchUpResult = std::result::Result; + +impl CatchUpLease { + /// Acquire the mount-level Exact lease for one logical backup id. + async fn acquire(store: Arc, backend_id: &str) -> Result { + let path = + format!("{MULTIWRITE_MOUNT_PREFIX}/{SYSTEM_DIR}/.multiwrite.catch-up.{backend_id}"); + let manager = store.pathlock_manager(); + let lease = manager + .acquire_exact( + &path, + manager.default_lock_timeout().max(Duration::from_secs(5)), + None, + ) + .await?; + Ok(Self { + store, + lease, + valid: AtomicBool::new(true), + }) + } + + /// Refresh the lease and return false for every non-refreshed result. + async fn refresh(&self) -> Result { + let result = self.store.pathlock_manager().refresh(&self.lease).await; + let refreshed = matches!(result.as_deref(), Ok("refreshed")); + if !refreshed { + self.valid.store(false, Ordering::SeqCst); + } + result.map(|_| refreshed).map_err(Error::from) + } + + /// Return whether every explicit refresh has retained the lease. + fn is_valid(&self) -> bool { + self.valid.load(Ordering::SeqCst) + } + + /// Release this worker's Exact lease. + async fn release(&self) -> Result<()> { + self.store + .pathlock_manager() + .release(&self.lease) + .await + .map_err(Error::from) + } +} + +impl CatchUpWorker { + /// Create one worker for an encrypted primary and backup handle. + pub fn new( + backend_id: String, + primary: Arc, + backup: Arc, + store: Arc, + provider: Arc, + ) -> Self { + Self { + backend_id, + primary, + backup, + store, + provider, + checkpoint_consumer: Arc::new(EmptyCheckpointConsumer), + cancellation: None, + } + } + + /// Attach the parsed-checkpoint consumer supplied by T-011. + pub fn with_checkpoint_consumer( + mut self, + checkpoint_consumer: Arc, + ) -> Self { + self.checkpoint_consumer = checkpoint_consumer; + self + } + + /// Attach a runtime cancellation receiver for interruptible catch-up rounds. + pub(crate) fn with_cancellation(mut self, cancellation: watch::Receiver) -> Self { + self.cancellation = Some(cancellation); + self + } + + /// Acquire the backup lease, execute one bounded round, and release it. + pub async fn run_once(&self) -> Result<()> { + let outcome: CatchUpResult<()> = async { + let lease = + Arc::new(CatchUpLease::acquire(self.store.clone(), &self.backend_id).await?); + debug!(backend = %self.backend_id, "multi-write catch-up lease acquired"); + let operation = self.run_round(&lease).await; + let release = lease.release().await; + match (operation, release) { + (Ok(()), Ok(())) => Ok(()), + (Err(CatchUpFailure::Cancelled), Ok(())) => Err(CatchUpFailure::Cancelled), + (Err(CatchUpFailure::Failed(error)), _) => Err(CatchUpFailure::Failed(error)), + (_, Err(error)) => Err(CatchUpFailure::Failed(error)), + } + } + .await; + match outcome { + Ok(()) => Ok(()), + Err(CatchUpFailure::Cancelled) => Err(Error::internal("catch-up was cancelled")), + Err(CatchUpFailure::Failed(error)) => { + self.store.record_worker_error(MultiWriteWorker::CatchUp); + Err(error) + } + } + } + + /// Wait until runtime cancellation or remain pending for standalone workers. + async fn cancelled(&self) { + let Some(mut cancellation) = self.cancellation.clone() else { + std::future::pending::<()>().await; + return; + }; + while !*cancellation.borrow() { + if cancellation.changed().await.is_err() { + return; + } + } + } + + /// Reconcile registry membership and one fixed partition window per scope. + async fn run_round(&self, lease: &Arc) -> CatchUpResult<()> { + let guarded = LeaseGuardedFileSystem { + inner: self.backup.clone(), + lease: lease.clone(), + }; + let mut accounts = self.read_accounts().await?; + self.remove_extra_accounts(&guarded, &accounts).await?; + if !accounts.iter().any(|account| account == SYSTEM_DIR) { + accounts.insert(0, SYSTEM_DIR.to_string()); + } + + for account in accounts { + let path = self.store.paths().account_manifest(&account)?.0; + let manifest: PartitionsManifest = match self.store.read_json(&path).await { + Ok(manifest) => manifest, + Err(Error::NotFound(_)) => continue, + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::CatchUp); + warn!(backend = %self.backend_id, account = %account, error = %error, + "multi-write catch-up account metadata unavailable"); + continue; + } + }; + if let Err(error) = manifest.validate() { + self.store.record_worker_error(MultiWriteWorker::CatchUp); + warn!(backend = %self.backend_id, account = %account, error = %error, + "multi-write catch-up account metadata invalid"); + continue; + } + for (&partition_id, partition) in &manifest.partitions { + if partition.state != PartitionState::Stable { + continue; + } + let scope = ScopeKey { + account_id: account.clone(), + partition_id, + epoch: manifest.epoch, + }; + if let Err(failure) = self.reconcile_scope(&guarded, lease, &scope).await { + let error = match failure { + CatchUpFailure::Cancelled => return Err(CatchUpFailure::Cancelled), + CatchUpFailure::Failed(Error::NotFound(_)) => continue, + CatchUpFailure::Failed(error) => error, + }; + if !lease.is_valid() { + return Err(error.into()); + } + self.store.record_worker_error(MultiWriteWorker::CatchUp); + warn!( + backend = %self.backend_id, + account = %account, + partition = partition_id, + error = %error, + "multi-write catch-up partition failed" + ); + } + } + } + Ok(()) + } + + /// Read sorted account ids from the encrypted mount-level registry. + async fn read_accounts(&self) -> Result> { + let bytes = self + .with_context( + SYSTEM_DIR, + self.primary.read("/_system/accounts.json", 0, 0), + ) + .await?; + let value: Value = serde_json::from_slice(&bytes)?; + let accounts = value + .get("accounts") + .and_then(Value::as_object) + .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?; + let mut ids = accounts.keys().cloned().collect::>(); + ids.sort(); + Ok(ids) + } + + /// Remove backup account roots that are absent from the primary registry. + async fn remove_extra_accounts( + &self, + backup: &LeaseGuardedFileSystem, + accounts: &[String], + ) -> Result<()> { + let expected = accounts.iter().map(String::as_str).collect::>(); + let entries = self + .with_context(SYSTEM_DIR, backup.read_internal_dir("/")) + .await?; + for entry in entries { + if entry.is_dir && entry.name != SYSTEM_DIR && !expected.contains(entry.name.as_str()) { + let path = format!("/{}", entry.name); + self.with_context(&entry.name, backup.remove_all(&path)) + .await?; + } + } + Ok(()) + } + + /// Consume one fixed `(synced_seq, tail]` window and advance after convergence. + async fn reconcile_scope( + &self, + backup: &LeaseGuardedFileSystem, + lease: &CatchUpLease, + scope: &ScopeKey, + ) -> CatchUpResult<()> { + let manifest = self.provider.read_manifest(scope).await?; + let state = manifest + .backend_states + .get(&self.backend_id) + .ok_or_else(|| { + Error::invalid_operation(format!("unknown backend id: {}", self.backend_id)) + })?; + let tail = manifest.next_seq - 1; + debug!( + backend = %self.backend_id, + account = %scope.account_id, + partition = scope.partition_id, + synced_seq = state.synced_seq, + tail_seq = tail, + lag = tail - state.synced_seq, + "multi-write catch-up partition snapshot" + ); + if state.synced_seq == tail { + return Ok(()); + } + let records = self + .provider + .read_committed_head(scope, state.synced_seq + 1, tail + 1) + .await; + let records = match records { + Ok(records) if continuous_records(&records, state.synced_seq + 1, tail + 1) => { + Some(records) + } + _ => self + .provider + .read_committed_range(scope, state.synced_seq + 1, tail + 1) + .await + .ok() + .filter(|records| continuous_records(records, state.synced_seq + 1, tail + 1)), + }; + let Some(records) = records else { + return self + .reconcile_checkpoint(backup, lease, scope, state.synced_seq, tail) + .await; + }; + let progress = self + .reconcile_records(backup, lease, scope, state.synced_seq, tail, records) + .await?; + self.advance_progress(lease, scope, state.synced_seq, progress) + .await?; + Ok(()) + } + + /// Reconcile ordinary paths first, then delayed exact and prefix directory paths. + async fn reconcile_records( + &self, + backup: &LeaseGuardedFileSystem, + lease: &CatchUpLease, + scope: &ScopeKey, + synced_seq: u64, + tail: u64, + records: Vec, + ) -> CatchUpResult { + let (mut ordinary, mut delayed_exact, mut prefixes, mut scope_deletes) = ( + BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), + ); + for record in records { + match record.event_type { + SegmentEventType::Write | SegmentEventType::Remove => { + let path = self + .store + .paths() + .backend_path(&scope.account_id, &record.path)?; + insert_earliest(&mut ordinary, path, record.seq); + } + SegmentEventType::RemoveTree => { + let path = self + .store + .paths() + .backend_path(&scope.account_id, &record.path)?; + insert_earliest(&mut prefixes, path, record.seq); + } + SegmentEventType::MoveTree => { + let source = self + .store + .paths() + .backend_path(&scope.account_id, &record.path)?; + let destination = self.store.paths().backend_path( + &scope.account_id, + record.destination_path.as_deref().ok_or_else(|| { + Error::Serialization("move marker lacks destination".to_string()) + })?, + )?; + let (exact, dirty) = self + .with_context( + &scope.account_id, + classify_move_paths(backup, &source, &destination), + ) + .await?; + for path in exact { + insert_earliest(&mut delayed_exact, path, record.seq); + } + for path in dirty { + insert_earliest(&mut prefixes, path, record.seq); + } + } + SegmentEventType::DeleteScope => { + if scope.account_id != SYSTEM_DIR { + return Err(Error::invalid_operation( + "DeleteScope is valid only in the system scope", + ) + .into()); + } + let (account_id, path) = + delete_scope_target(self.store.as_ref(), &record.path)?; + insert_earliest_scope_delete(&mut scope_deletes, path, account_id, record.seq); + } + } + } + let mut failed_from = None; + for (path, (account_id, seq)) in scope_deletes { + match self + .remove_scope_with_retry(backup, lease, &account_id, &path) + .await + { + Ok(()) => { + if let Err(error) = self + .reset_account_progress_after_scope_delete(lease, &account_id) + .await + { + if !lease.is_valid() { + return Err(error.into()); + } + warn!( + backend = %self.backend_id, + account = %account_id, + error = %error, + "multi-write catch-up scope-delete progress reset failed" + ); + failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq))); + } + } + Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled), + Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => { + return Err(Error::internal("catch-up lease was not refreshed").into()); + } + Err(CatchUpFailure::Failed(_)) => { + failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq))); + } + } + } + for (paths, prefix) in [(ordinary, false), (delayed_exact, false), (prefixes, true)] { + for (path, seq) in paths { + match self + .reconcile_with_retry(backup, lease, &scope.account_id, &path, prefix) + .await + { + Ok(()) => {} + Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled), + Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => { + return Err(Error::internal("catch-up lease was not refreshed").into()); + } + Err(CatchUpFailure::Failed(_)) => { + failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq))); + } + } + } + } + Ok(failed_from.map_or(tail, |seq| seq.saturating_sub(1).max(synced_seq))) + } + + /// Remove one backup scope root with the same bounded retry policy as path sync. + async fn remove_scope_with_retry( + &self, + backup: &LeaseGuardedFileSystem, + lease: &CatchUpLease, + account_id: &str, + path: &str, + ) -> CatchUpResult<()> { + let mut last_error = None; + for attempt in 0..=3 { + match self + .with_context(account_id, remove_if_present(backup, path)) + .await + { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), + } + if !lease.is_valid() || attempt == 3 { + break; + } + let jitter = rand::thread_rng().gen_range(0..=50); + let delay_ms = (1u64 << attempt) * 1_000 + jitter; + warn!( + backend = %self.backend_id, + account = %account_id, + path = %path, + retry = attempt + 1, + delay_ms, + error = %last_error.as_ref().unwrap(), + "multi-write catch-up scope-delete retry" + ); + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {} + _ = self.cancelled() => return Err(CatchUpFailure::Cancelled), + } + } + Err(CatchUpFailure::Failed(last_error.unwrap())) + } + + /// Reset this backup's progress for a recreated account after scope delete. + async fn reset_account_progress_after_scope_delete( + &self, + lease: &CatchUpLease, + account_id: &str, + ) -> Result<()> { + let path = self.store.paths().account_manifest(account_id)?.0; + let manifest: PartitionsManifest = match self.store.read_json(&path).await { + Ok(manifest) => manifest, + Err(Error::NotFound(_)) => return Ok(()), + Err(error) => return Err(error), + }; + manifest.validate()?; + if !lease.refresh().await? { + return Err(Error::internal("catch-up lease was not refreshed")); + } + for (&partition_id, _) in &manifest.partitions { + let scope = ScopeKey { + account_id: account_id.to_string(), + partition_id, + epoch: manifest.epoch, + }; + self.provider + .advance_backend_state(&scope, &self.backend_id, 0) + .await?; + } + Ok(()) + } + + /// Refresh the lease immediately before a monotonic progress update. + async fn advance_progress( + &self, + lease: &CatchUpLease, + scope: &ScopeKey, + previous: u64, + progress: u64, + ) -> Result<()> { + if progress == previous { + return Ok(()); + } + if !lease.refresh().await? { + return Err(Error::internal("catch-up lease was not refreshed")); + } + self.provider + .advance_backend_state(scope, &self.backend_id, progress) + .await + } + + /// Reconcile one path with fixed exponential delays and bounded jitter. + async fn reconcile_with_retry( + &self, + backup: &LeaseGuardedFileSystem, + lease: &CatchUpLease, + account_id: &str, + path: &str, + prefix: bool, + ) -> CatchUpResult<()> { + let mut last_error = None; + for attempt in 0..=3 { + let operation = if prefix { + self.with_context( + account_id, + reconcile_prefix(self.primary.as_ref(), backup, path), + ) + .await + } else { + self.with_context( + account_id, + reconcile_file(self.primary.as_ref(), backup, path), + ) + .await + }; + match operation { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), + } + if !lease.is_valid() || attempt == 3 { + break; + } + let jitter = rand::thread_rng().gen_range(0..=50); + let delay_ms = (1u64 << attempt) * 1_000 + jitter; + warn!( + backend = %self.backend_id, + account = %account_id, + path = %path, + retry = attempt + 1, + delay_ms, + error = %last_error.as_ref().unwrap(), + "multi-write catch-up path retry" + ); + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {} + _ = self.cancelled() => return Err(CatchUpFailure::Cancelled), + } + } + Err(CatchUpFailure::Failed(last_error.unwrap())) + } + + /// Consume one already parsed checkpoint and advance only after full convergence. + async fn reconcile_checkpoint( + &self, + backup: &LeaseGuardedFileSystem, + lease: &CatchUpLease, + scope: &ScopeKey, + synced_seq: u64, + tail: u64, + ) -> CatchUpResult<()> { + let snapshot = match self + .checkpoint_consumer + .read_latest(scope, synced_seq, tail) + .await? + { + CheckpointReadResult::Snapshot(snapshot) => snapshot, + CheckpointReadResult::NoCheckpoint => { + return Err(Error::invalid_operation("segment window is not continuous").into()); + } + CheckpointReadResult::RetryLatestCheckpoint => { + return Err(Error::invalid_operation("retry latest checkpoint").into()); + } + }; + if snapshot.checkpoint_to_seq <= synced_seq || snapshot.checkpoint_to_seq > tail { + return Err(Error::Serialization( + "checkpoint sequence is outside the fixed catch-up window".to_string(), + ) + .into()); + } + let mut paths = BTreeSet::new(); + for state in snapshot.file_states { + state.validate(scope)?; + if state.latest_seq.seq > snapshot.checkpoint_to_seq { + return Err(Error::Serialization( + "checkpoint file state exceeds checkpoint sequence".to_string(), + ) + .into()); + } + paths.insert( + self.store + .paths() + .backend_path(&scope.account_id, &state.path)?, + ); + } + for path in paths { + self.reconcile_with_retry(backup, lease, &scope.account_id, &path, false) + .await?; + } + let mut prefixes = BTreeSet::new(); + for prefix in snapshot.directory_prefixes { + prefixes.insert( + self.store + .paths() + .backend_path(&scope.account_id, &prefix)?, + ); + } + for prefix in prefixes { + self.reconcile_with_retry(backup, lease, &scope.account_id, &prefix, true) + .await?; + } + self.advance_progress(lease, scope, synced_seq, snapshot.checkpoint_to_seq) + .await?; + if snapshot.checkpoint_to_seq == tail { + return Ok(()); + } + let records = self + .provider + .read_committed_range(scope, snapshot.checkpoint_to_seq + 1, tail + 1) + .await?; + if !continuous_records(&records, snapshot.checkpoint_to_seq + 1, tail + 1) { + return Err(Error::invalid_operation( + "post-checkpoint segment window is not continuous", + ) + .into()); + } + let progress = self + .reconcile_records( + backup, + lease, + scope, + snapshot.checkpoint_to_seq, + tail, + records, + ) + .await?; + self.advance_progress(lease, scope, snapshot.checkpoint_to_seq, progress) + .await?; + Ok(()) + } + + /// Run one filesystem future with cache bypass and automatic PathLock disabled. + async fn with_context( + &self, + account_id: &str, + future: impl Future>, + ) -> Result { + let context: FsContext = Arc::new( + FsContextInner::new(account_id) + .with_bypass_cache(true) + .with_auto_pathlock_disabled(), + ); + FS_CTX.scope(context, future).await + } +} + +/// Insert one path while retaining its earliest dependent sequence. +fn insert_earliest(paths: &mut BTreeMap, path: String, seq: u64) { + paths + .entry(path) + .and_modify(|current| *current = (*current).min(seq)) + .or_insert(seq); +} + +/// Insert one scope delete while retaining its earliest dependent sequence. +fn insert_earliest_scope_delete( + paths: &mut BTreeMap, + path: String, + account_id: String, + seq: u64, +) { + paths + .entry(path) + .and_modify(|(_, current)| *current = (*current).min(seq)) + .or_insert((account_id, seq)); +} + +/// Convert a logical `/local/{scope}` deletion target into backup path and context. +fn delete_scope_target(store: &MetadataStore, logical_path: &str) -> Result<(String, String)> { + let path = store.paths().raw_backend_path(logical_path)?; + let account_id = path + .trim_start_matches('/') + .split('/') + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| Error::invalid_path(format!("invalid DeleteScope path: {logical_path}")))?; + Ok((account_id.to_string(), path)) +} + +struct LeaseGuardedFileSystem { + inner: Arc, + lease: Arc, +} + +impl LeaseGuardedFileSystem { + /// Reject the next backup mutation unless the lease refreshes literally. + async fn before_write(&self) -> Result<()> { + if self.lease.refresh().await? { + Ok(()) + } else { + Err(Error::internal("catch-up lease was not refreshed")) + } + } +} + +#[async_trait] +impl FileSystem for LeaseGuardedFileSystem { + /// Refresh the catch-up lease before creating a backup file. + async fn create(&self, path: &str) -> Result<()> { + self.before_write().await?; + self.inner.create(path).await + } + + /// Refresh the catch-up lease before creating a backup directory. + async fn mkdir(&self, path: &str, mode: u32) -> Result<()> { + self.before_write().await?; + self.inner.mkdir(path, mode).await + } + + /// Refresh the catch-up lease before removing a backup file. + async fn remove(&self, path: &str) -> Result<()> { + self.before_write().await?; + self.inner.remove(path).await + } + + /// Refresh the catch-up lease before removing a backup subtree. + async fn remove_all(&self, path: &str) -> Result<()> { + self.before_write().await?; + self.inner.remove_all(path).await + } + + /// Read bytes from the guarded backup without mutating it. + async fn read(&self, path: &str, offset: u64, size: u64) -> Result> { + self.inner.read(path, offset, size).await + } + + /// Refresh the catch-up lease before writing backup bytes. + async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result { + self.before_write().await?; + self.inner.write(path, data, offset, flags).await + } + + /// List guarded backup entries without mutating them. + async fn read_dir( + &self, + path: &str, + offset: Option, + limit: Option, + sort_by: Option, + sort_order: Option, + ) -> Result> { + self.inner + .read_dir(path, offset, limit, sort_by, sort_order) + .await + } + + /// Read guarded backup metadata without mutating it. + async fn stat(&self, path: &str) -> Result { + self.inner.stat(path).await + } + + /// Refresh the catch-up lease before renaming a backup path. + async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> { + self.before_write().await?; + self.inner.rename(old_path, new_path).await + } + + /// Refresh the catch-up lease before changing a backup mode. + async fn chmod(&self, path: &str, mode: u32) -> Result<()> { + self.before_write().await?; + self.inner.chmod(path, mode).await + } +} + +/// Return whether records exactly cover one requested half-open sequence range. +fn continuous_records(records: &[SegmentRecord], from: u64, to: u64) -> bool { + records.len() as u64 == to - from + && records + .iter() + .enumerate() + .all(|(index, record)| record.seq == from + index as u64) +} + +/// Classify move source and destination by the backup's current source type. +pub async fn classify_move_paths( + backup: &dyn FileSystem, + source: &str, + destination: &str, +) -> Result<(HashSet, HashSet)> { + let paths = HashSet::from([source.to_string(), destination.to_string()]); + match backup.stat(source).await { + Ok(info) if !info.is_dir => Ok((paths, HashSet::new())), + Ok(_) | Err(Error::NotFound(_)) => Ok((HashSet::new(), paths)), + Err(error) => Err(error), + } +} + +/// Reconcile one file or directory prefix from primary state into a backup. +pub async fn reconcile_prefix( + primary: &dyn FileSystem, + backup: &dyn FileSystem, + prefix: &str, +) -> Result<()> { + if is_catch_up_internal_path(prefix) { + return Ok(()); + } + let root = match primary.stat(prefix).await { + Ok(root) => root, + Err(Error::NotFound(_)) => { + return remove_stale_paths(backup, prefix, &HashSet::new()).await; + } + Err(error) => return Err(error), + }; + if !root.is_dir { + return copy_file_state(primary, backup, prefix).await; + } + + let entries = primary + .tree_directory(prefix, true, None, None, None, None, None, false) + .await?; + ensure_directory(backup, prefix, root.mode).await?; + let mut current = HashSet::from([prefix.to_string()]); + for entry in entries + .iter() + .filter(|entry| !is_catch_up_internal_path(&entry.path)) + { + current.insert(entry.path.clone()); + if entry.info.is_dir { + ensure_directory(backup, &entry.path, entry.info.mode).await?; + } else { + copy_file_state(primary, backup, &entry.path).await?; + } + } + + remove_stale_paths(backup, prefix, ¤t).await +} + +/// Reconcile one exact path from the primary's current state. +pub async fn reconcile_file( + primary: &dyn FileSystem, + backup: &dyn FileSystem, + path: &str, +) -> Result<()> { + if is_catch_up_internal_path(path) { + return Ok(()); + } + match primary.stat(path).await { + Ok(info) if info.is_dir => reconcile_prefix(primary, backup, path).await, + Ok(_) => copy_file_state(primary, backup, path).await, + Err(Error::NotFound(_)) => remove_if_present(backup, path).await, + Err(error) => Err(error), + } +} + +/// Remove stale business paths while preserving internal files and their ancestors. +async fn remove_stale_paths( + backup: &dyn FileSystem, + prefix: &str, + current: &HashSet, +) -> Result<()> { + match backup.stat(prefix).await { + Ok(info) if !info.is_dir => { + return if current.contains(prefix) || is_multiwrite_internal_path(prefix) { + Ok(()) + } else { + remove_if_present(backup, prefix).await + }; + } + Ok(_) => {} + Err(Error::NotFound(_)) => return Ok(()), + Err(error) => return Err(error), + } + let entries = match backup + .tree_directory(prefix, true, None, None, None, None, None, false) + .await + { + Ok(entries) => entries, + Err(Error::NotFound(_)) => return Ok(()), + Err(error) => return Err(error), + }; + let protected = entries + .iter() + .filter(|entry| is_catch_up_internal_path(&entry.path)) + .flat_map(|entry| Path::new(&entry.path).ancestors().skip(1)) + .filter_map(Path::to_str) + .map(str::to_string) + .collect::>(); + let mut stale = std::iter::once(prefix.to_string()) + .chain(entries.into_iter().map(|entry| entry.path)) + .filter(|path| { + !is_catch_up_internal_path(path) && !current.contains(path) && !protected.contains(path) + }) + .collect::>(); + stale.sort_by_key(|path| std::cmp::Reverse(path.matches('/').count())); + for path in stale { + remove_if_present(backup, &path).await?; + } + Ok(()) +} + +/// Recognize internal paths in either logical or mount-relative path space. +fn is_catch_up_internal_path(path: &str) -> bool { + is_multiwrite_internal_path(path) + || is_multiwrite_internal_path(&format!( + "{MULTIWRITE_MOUNT_PREFIX}/{}", + path.trim_start_matches('/') + )) +} + +/// Ensure one backup path is a directory with the requested mode. +async fn ensure_directory(backup: &dyn FileSystem, path: &str, mode: u32) -> Result<()> { + match backup.stat(path).await { + Ok(info) if info.is_dir => {} + Ok(_) => { + backup.remove(path).await?; + backup.mkdir(path, mode).await?; + } + Err(Error::NotFound(_)) => { + backup.ensure_parent_dirs(path, 0o755).await?; + backup.mkdir(path, mode).await?; + } + Err(error) => return Err(error), + } + Ok(()) +} + +/// Replace one backup file with the primary bytes. +async fn copy_file_state( + primary: &dyn FileSystem, + backup: &dyn FileSystem, + path: &str, +) -> Result<()> { + if backup.stat(path).await.is_ok_and(|current| current.is_dir) { + backup.remove_all(path).await?; + } + let bytes = match primary.read(path, 0, 0).await { + Ok(bytes) => bytes, + Err(Error::NotFound(_)) => return remove_if_present(backup, path).await, + Err(error) => return Err(error), + }; + backup.ensure_parent_dirs(path, 0o755).await?; + backup.write(path, &bytes, 0, WriteFlag::Create).await?; + Ok(()) +} + +/// Remove one backup path while accepting an already converged absence. +async fn remove_if_present(backup: &dyn FileSystem, path: &str) -> Result<()> { + let result = match backup.stat(path).await { + Ok(info) if info.is_dir => backup.remove_all(path).await, + Ok(_) => backup.remove(path).await, + Err(Error::NotFound(_)) => return Ok(()), + Err(error) => return Err(error), + }; + match result { + Ok(()) | Err(Error::NotFound(_)) => Ok(()), + Err(error) => Err(error), + } +} diff --git a/crates/ragfs/src/multibackend/checkpoint.rs b/crates/ragfs/src/multibackend/checkpoint.rs new file mode 100644 index 0000000000..85f0987b38 --- /dev/null +++ b/crates/ragfs/src/multibackend/checkpoint.rs @@ -0,0 +1,1751 @@ +//! Checkpoint construction, validation, publication, and reading. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use tokio::sync::watch; +use tracing::warn; + +use crate::core::context::{FsContextInner, FS_CTX}; +use crate::core::errors::{Error, Result}; +use crate::core::filesystem::FileSystem; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::multibackend::catch_up::{CheckpointConsumer, CheckpointReadResult, CheckpointSnapshot}; +use crate::multibackend::codec::{decode_checkpoint_chunk, encode_checkpoint_chunk, sha256_hex}; +use crate::multibackend::constants::{ + CHECKPOINT_CHUNK_FILE_EXTENSION, CHECKPOINT_CHUNK_FILE_PREFIX, CHECKPOINT_DIR_PREFIX, + CHECKPOINT_DIR_TIME_FORMAT, MANIFEST_FILE, MAX_CHECKPOINT_NODES, MULTIWRITE_MOUNT_PREFIX, +}; +use crate::multibackend::gc::MetadataGc; +use crate::multibackend::meta::{MetadataStore, MultiWriteWorker}; +use crate::multibackend::model::{ + is_checkpoint_directory_name, CheckpointManifest, CheckpointNode, CheckpointsManifest, + ChunkDescriptor, DirectoryOperation, FileState, LatestCheckpoint, PartitionsManifest, + ProtocolStatus, ScopeKey, ScopedSeq, SegmentEventType, SegmentRecord, +}; +use crate::multibackend::provider::MultiWriteProvider; +use crate::multibackend::router::AccountRouter; + +/// Periodically builds checkpoints for every Stable account partition. +pub struct CheckpointWorker { + store: Arc, + builder: CheckpointBuilder, + gc: MetadataGc, +} +impl CheckpointWorker { + /// Create a worker over the production primary, store, and provider. + pub fn new( + primary: Arc, + store: Arc, + provider: Arc, + ) -> Self { + let builder = CheckpointBuilder::new(primary, store.clone(), provider.clone()); + Self { + gc: MetadataGc::new(store.clone(), provider), + builder, + store, + } + } + + /// Run checkpoint rounds at the configured interval until cancellation. + pub async fn run(&self, interval: Duration, mut cancellation: watch::Receiver) { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = ticker.tick() => if let Err(error) = self.run_once(interval).await { + warn!(error = %error, "multi-write checkpoint discovery failed"); + }, + changed = cancellation.changed() => { + if changed.is_err() || *cancellation.borrow() { + break; + } + } + } + } + } + + /// Build one checkpoint round while isolating account and scope failures. + pub async fn run_once(&self, interval: Duration) -> Result<()> { + let result = self.run_once_inner(interval).await; + if result.is_err() { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + } + result + } + + /// Execute one checkpoint round and return discovery failures. + async fn run_once_inner(&self, interval: Duration) -> Result<()> { + if self.store.protocol_status().await? != ProtocolStatus::Stable { + return Ok(()); + } + let now = now_ns()?; + let interval_ns = u64::try_from(interval.as_nanos()).unwrap_or(u64::MAX); + for account in self.store.initialized_accounts().await? { + let path = self.store.paths().account_manifest(&account)?.0; + let manifest: PartitionsManifest = match self.store.read_json(&path).await { + Ok(manifest) => manifest, + Err(Error::NotFound(_)) => continue, + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + warn!(account = %account, error = %error, + "multi-write checkpoint account metadata unavailable"); + continue; + } + }; + if let Err(error) = manifest.validate() { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + warn!(account = %account, error = %error, + "multi-write checkpoint account metadata invalid"); + continue; + } + for (&partition_id, partition) in &manifest.partitions { + if partition.state != crate::multibackend::model::PartitionState::Stable { + continue; + } + let scope = ScopeKey { + account_id: account.clone(), + partition_id, + epoch: manifest.epoch, + }; + let pointer_path = self + .store + .paths() + .checkpoints_manifest(&account, partition_id)? + .0; + let checkpoints: CheckpointsManifest = + match self.store.read_json(&pointer_path).await { + Ok(checkpoints) => checkpoints, + Err(Error::NotFound(_)) => continue, + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + warn!(account = %account, partition = partition_id, error = %error, + "multi-write checkpoint pointer unavailable"); + continue; + } + }; + if let Err(error) = checkpoints.validate() { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + warn!(account = %account, partition = partition_id, error = %error, + "multi-write checkpoint pointer invalid"); + continue; + } + if checkpoints + .latest_checkpoint + .as_ref() + .is_some_and(|latest| now < latest.updated_at_ns.saturating_add(interval_ns)) + { + continue; + } + match self.builder.build_scope(&scope).await { + Ok(CheckpointBuildResult::Published(_)) => { + if let Err(error) = self.gc.collect_scope(&scope).await { + warn!(account = %account, partition = partition_id, error = %error, + "multi-write metadata garbage collection failed"); + } + } + Ok(_) => {} + Err(Error::NotFound(_)) => {} + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Checkpoint); + warn!(account = %account, partition = partition_id, error = %error, + "multi-write checkpoint build failed"); + } + } + } + } + Ok(()) + } +} + +/// Result of one bounded checkpoint build attempt. +pub enum CheckpointBuildResult { + /// No new sealed sequence exists for this scope. + NoSealedChanges, + /// A validated checkpoint became the latest publication. + Published(LatestCheckpoint), + /// Another builder published first and this candidate was discarded. + Retry, +} + +#[derive(Default)] +struct SegmentDelta { + states: BTreeMap, + exact_paths: BTreeSet, + prefixes: BTreeSet, +} + +impl SegmentDelta { + /// Return whether this segment delta cannot change checkpoint contents. + fn is_empty(&self) -> bool { + self.states.is_empty() && self.exact_paths.is_empty() && self.prefixes.is_empty() + } +} + +enum ChunkMergeTarget<'a> { + All, + Range(&'a str, Option<&'a str>), + RootOnly(&'a str), +} + +/// Builds immutable checkpoints from sealed records and current primary state. +pub struct CheckpointBuilder { + primary: Arc, + store: Arc, + provider: Arc, + router: AccountRouter, +} +impl CheckpointBuilder { + /// Create a builder over the encrypted primary and raw metadata store. + pub fn new( + primary: Arc, + store: Arc, + provider: Arc, + ) -> Self { + Self { + primary, + router: AccountRouter::new(store.clone()), + store, + provider, + } + } + + /// Build and compare-before-publish one Stable partition checkpoint. + pub async fn build_scope(&self, scope: &ScopeKey) -> Result { + scope.validate()?; + let pointer_path = self + .store + .paths() + .checkpoints_manifest(&scope.account_id, scope.partition_id)? + .0; + let lease = self + .store + .pathlock_manager() + .acquire_exact( + &pointer_path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(Duration::from_secs(5)), + None, + ) + .await?; + let read_pointer = async { + let start: CheckpointsManifest = self.store.read_json(&pointer_path).await?; + start.validate()?; + Ok(start) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + let start = finish_with_release(read_pointer, release)?; + let segments = self.provider.read_manifest(scope).await?; + segments.validate()?; + let Some(tail) = segments + .segments + .iter() + .filter_map(|segment| segment.segment_to_seq) + .last() + else { + return Ok(CheckpointBuildResult::NoSealedChanges); + }; + let old_to = start + .latest_checkpoint + .as_ref() + .map_or(0, |latest| latest.checkpoint_to_seq); + if tail <= old_to { + return Ok(CheckpointBuildResult::NoSealedChanges); + } + let min_synced = segments + .backend_states + .values() + .map(|state| state.synced_seq) + .min(); + let created_at_ns = now_ns()?; + let candidate = checkpoint_candidate_name(created_at_ns)?; + let directory = self + .store + .paths() + .checkpoints_dir(&scope.account_id, scope.partition_id)? + .0; + let candidate_path = format!("{directory}/{}", candidate.trim_end_matches('/')); + let candidate_lease = self + .store + .pathlock_manager() + .acquire_exact( + &candidate_path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(Duration::from_secs(5)), + None, + ) + .await?; + let operation = async { + let mut manifest = self + .prepare_candidate_checkpoint(scope, &start, &candidate, created_at_ns, old_to) + .await?; + let mut chunk_index = checkpoint_chunk_count(&manifest.root); + for descriptor in segments + .segments + .iter() + .filter(|descriptor| descriptor.segment_to_seq.is_some_and(|to| to > old_to)) + { + let records = self.provider.read_sealed_segment(scope, descriptor).await?; + let records = records + .into_iter() + .filter(|record| record.seq > old_to && record.seq <= tail) + .collect::>(); + if records.is_empty() { + continue; + } + let delta = self.segment_delta(scope, records).await?; + self.apply_checkpoint_delta( + scope, + &candidate, + &mut manifest, + &delta, + &mut chunk_index, + ) + .await?; + } + self.prune_checkpoint_tombstones( + scope, + &candidate, + &mut manifest, + min_synced, + &mut chunk_index, + ) + .await?; + manifest.version = 1; + manifest.partition_id = scope.partition_id; + manifest.epoch = scope.epoch; + manifest.checkpoint_from_seq = old_to + 1; + manifest.checkpoint_to_seq = tail; + manifest.created_at_ns = created_at_ns; + manifest.checksum = tree_checksum(&manifest.root)?; + manifest.validate_for_scope(scope)?; + let manifest_bytes = serde_json::to_vec(&manifest)?; + self.store + .write_bytes( + &format!("{directory}/{candidate}{MANIFEST_FILE}"), + &manifest_bytes, + ) + .await?; + verify_checkpoint(self.store.as_ref(), scope, &manifest, &candidate, tail).await?; + let latest = LatestCheckpoint { + path: candidate, + checkpoint_to_seq: tail, + updated_at_ns: created_at_ns, + }; + let lease = self + .store + .pathlock_manager() + .acquire_exact( + &pointer_path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(Duration::from_secs(5)), + None, + ) + .await?; + let publish = async { + let current: CheckpointsManifest = self.store.read_json(&pointer_path).await?; + if current.latest_checkpoint != start.latest_checkpoint { + return Ok(CheckpointBuildResult::Retry); + } + self.store + .publish_json( + &pointer_path, + &CheckpointsManifest { + version: 1, + latest_checkpoint: Some(latest.clone()), + }, + CheckpointsManifest::validate, + ) + .await?; + Ok(CheckpointBuildResult::Published(latest)) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + finish_with_release(publish, release) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&candidate_lease) + .await + .map_err(Error::from); + finish_with_release(operation, release) + } + + /// Prepare a candidate checkpoint directory from the latest checkpoint or an empty base. + async fn prepare_candidate_checkpoint( + &self, + scope: &ScopeKey, + start: &CheckpointsManifest, + candidate: &str, + created_at_ns: u64, + old_to: u64, + ) -> Result { + if let Some(latest) = &start.latest_checkpoint { + let manifest = read_manifest(self.store.as_ref(), scope, &latest.path).await?; + if manifest.checkpoint_to_seq != latest.checkpoint_to_seq { + return Err(Error::Serialization( + "checkpoint pointer sequence mismatch".into(), + )); + } + let manifest = copy_checkpoint_files( + self.store.as_ref(), + scope, + &latest.path, + candidate, + &manifest, + ) + .await?; + return Ok(manifest); + } + + let mut chunk_index = 0; + let (root, chunks) = encode_states_with_index(scope, Vec::new(), &mut chunk_index)?; + let checksum = tree_checksum(&root)?; + let manifest = CheckpointManifest { + version: 1, + partition_id: scope.partition_id, + epoch: scope.epoch, + checkpoint_from_seq: 1, + checkpoint_to_seq: old_to, + created_at_ns, + root, + checksum, + }; + manifest.validate_for_scope(scope)?; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?; + self.store + .write_bytes( + &format!("{directory}/{MANIFEST_FILE}"), + &serde_json::to_vec(&manifest)?, + ) + .await?; + Ok(manifest) + } + + /// Convert one sealed segment into checkpoint path replacements and prefix invalidations. + async fn segment_delta( + &self, + scope: &ScopeKey, + records: Vec, + ) -> Result { + let mut delta = SegmentDelta::default(); + for record in records { + match record.event_type { + SegmentEventType::Write | SegmentEventType::Remove => { + let backend_path = self + .store + .paths() + .backend_path(&scope.account_id, &record.path)?; + let deleted = match FS_CTX + .scope( + Arc::new( + FsContextInner::new(&scope.account_id) + .with_bypass_cache(true) + .with_auto_pathlock_disabled(), + ), + self.primary.stat(&backend_path), + ) + .await + { + Ok(_) => None, + Err(Error::NotFound(_)) => Some(true), + Err(error) => return Err(error), + }; + delta.exact_paths.insert(record.path.clone()); + delta.states.insert( + record.path.clone(), + FileState { + path: record.path, + latest_seq: ScopedSeq { + scope: scope.clone(), + seq: record.seq, + }, + deleted, + }, + ); + } + SegmentEventType::RemoveTree | SegmentEventType::MoveTree => { + let mut prefixes = vec![record.path.clone()]; + if let Some(destination) = record.destination_path { + prefixes.push(destination); + } + for prefix in prefixes { + delta.prefixes.insert(prefix.clone()); + delta + .states + .retain(|path, _| !is_path_within(path, &prefix)); + for path in self.scan_prefix(scope, &prefix).await? { + delta.states.insert( + path.clone(), + FileState { + path, + latest_seq: ScopedSeq { + scope: scope.clone(), + seq: record.seq, + }, + deleted: None, + }, + ); + } + } + } + SegmentEventType::DeleteScope => {} + } + } + Ok(delta) + } + + /// Apply one segment delta to the candidate manifest by rewriting only needed chunks. + async fn apply_checkpoint_delta( + &self, + scope: &ScopeKey, + candidate: &str, + manifest: &mut CheckpointManifest, + delta: &SegmentDelta, + chunk_index: &mut usize, + ) -> Result<()> { + if delta.is_empty() { + return Ok(()); + } + if manifest.root.chunks.is_empty() { + self.rewrite_full_checkpoint_chunk(scope, candidate, manifest, delta, chunk_index) + .await?; + } else { + self.rewrite_split_checkpoint_chunks(scope, candidate, manifest, delta, chunk_index) + .await?; + } + manifest.root.validate()?; + Ok(()) + } + + /// Rewrite a checkpoint whose complete state still lives in the root chunk. + async fn rewrite_full_checkpoint_chunk( + &self, + scope: &ScopeKey, + candidate: &str, + manifest: &mut CheckpointManifest, + delta: &SegmentDelta, + chunk_index: &mut usize, + ) -> Result<()> { + let old_root = manifest.root.clone(); + let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?; + let states = + read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true) + .await?; + let mut remaining = delta.states.clone(); + let states = merge_checkpoint_states(states, delta, &mut remaining, ChunkMergeTarget::All); + let (root, chunks) = encode_states_with_index(scope, states, chunk_index)?; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + manifest.root = root; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + checkpoint_chunk_files(&old_root), + &manifest.root, + ) + .await; + Ok(()) + } + + /// Rewrite affected child chunks and insert new states not covered by old ranges. + async fn rewrite_split_checkpoint_chunks( + &self, + scope: &ScopeKey, + candidate: &str, + manifest: &mut CheckpointManifest, + delta: &SegmentDelta, + chunk_index: &mut usize, + ) -> Result<()> { + let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?; + let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id); + let mut remaining = delta.states.clone(); + + if checkpoint_state_is_dirty(&account_root, delta) || remaining.contains_key(&account_root) + { + let old_root = manifest.root.clone(); + let children = std::mem::take(&mut manifest.root.chunks); + let states = read_checkpoint_chunk_states( + self.store.as_ref(), + scope, + &directory, + &old_root, + true, + ) + .await?; + let states = merge_checkpoint_states( + states, + delta, + &mut remaining, + ChunkMergeTarget::RootOnly(&account_root), + ); + let (mut root, chunks) = encode_states_with_index(scope, states, chunk_index)?; + root.chunks = children; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + manifest.root = root; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + vec![old_root.file], + &manifest.root, + ) + .await; + } + + let mut next_children = Vec::new(); + let children = std::mem::take(&mut manifest.root.chunks); + for descriptor in children { + if checkpoint_descriptor_is_dirty(&descriptor, delta)? { + let old_files = checkpoint_chunk_files(&descriptor); + let states = read_checkpoint_chunk_states( + self.store.as_ref(), + scope, + &directory, + &descriptor, + false, + ) + .await?; + let (start, end) = checkpoint_descriptor_range(&descriptor)?; + let states = merge_checkpoint_states( + states, + delta, + &mut remaining, + ChunkMergeTarget::Range(start, end), + ); + let (mut descriptors, chunks) = + encode_child_states_with_index(scope, states, start, end, chunk_index)?; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + let replacement_root = ChunkDescriptor { + file: String::new(), + root_path: account_root.clone(), + start_path: None, + end_path: None, + file_state_count: 0, + checksum: String::new(), + chunks: descriptors.clone(), + }; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + old_files, + &replacement_root, + ) + .await; + next_children.append(&mut descriptors); + } else { + next_children.push(descriptor); + } + } + + if !remaining.is_empty() { + return Err(Error::Serialization( + "checkpoint delta paths were not assigned to a chunk".into(), + )); + } + sort_checkpoint_children(&mut next_children, &account_root)?; + manifest.root.chunks = next_children; + Ok(()) + } + + /// Remove checkpoint tombstones that no current backend still needs. + async fn prune_checkpoint_tombstones( + &self, + scope: &ScopeKey, + candidate: &str, + manifest: &mut CheckpointManifest, + min_synced: Option, + chunk_index: &mut usize, + ) -> Result<()> { + if manifest.root.chunks.is_empty() { + let old_root = manifest.root.clone(); + let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?; + let states = read_checkpoint_chunk_states( + self.store.as_ref(), + scope, + &directory, + &old_root, + true, + ) + .await?; + let retained = retain_checkpoint_states(states.clone(), min_synced); + if retained != states { + let (root, chunks) = encode_states_with_index(scope, retained, chunk_index)?; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + manifest.root = root; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + checkpoint_chunk_files(&old_root), + &manifest.root, + ) + .await; + } + return Ok(()); + } + + let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?; + let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id); + let old_root = manifest.root.clone(); + let children = std::mem::take(&mut manifest.root.chunks); + let states = + read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true) + .await?; + let retained = retain_checkpoint_states(states.clone(), min_synced); + if retained != states { + let (mut root, chunks) = encode_states_with_index(scope, retained, chunk_index)?; + root.chunks = children; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + manifest.root = root; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + vec![old_root.file], + &manifest.root, + ) + .await; + } else { + manifest.root.chunks = children; + } + + let mut next_children = Vec::new(); + for descriptor in std::mem::take(&mut manifest.root.chunks) { + let old_files = checkpoint_chunk_files(&descriptor); + let states = read_checkpoint_chunk_states( + self.store.as_ref(), + scope, + &directory, + &descriptor, + false, + ) + .await?; + let retained = retain_checkpoint_states(states.clone(), min_synced); + if retained == states { + next_children.push(descriptor); + continue; + } + let (start, end) = checkpoint_descriptor_range(&descriptor)?; + let (mut descriptors, chunks) = + encode_child_states_with_index(scope, retained, start, end, chunk_index)?; + write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?; + let replacement_root = ChunkDescriptor { + file: String::new(), + root_path: account_root.clone(), + start_path: None, + end_path: None, + file_state_count: 0, + checksum: String::new(), + chunks: descriptors.clone(), + }; + remove_replaced_checkpoint_files( + self.store.as_ref(), + scope, + candidate, + old_files, + &replacement_root, + ) + .await; + next_children.append(&mut descriptors); + } + sort_checkpoint_children(&mut next_children, &account_root)?; + manifest.root.chunks = next_children; + Ok(()) + } + + /// Scan one current primary prefix and retain paths owned by this scope. + async fn scan_prefix(&self, scope: &ScopeKey, logical_prefix: &str) -> Result> { + let backend_prefix = self + .store + .paths() + .backend_path(&scope.account_id, logical_prefix)?; + let context = Arc::new( + FsContextInner::new(&scope.account_id) + .with_bypass_cache(true) + .with_auto_pathlock_disabled(), + ); + let paths = FS_CTX + .scope(context, async { + let root = match self.primary.stat(&backend_prefix).await { + Ok(root) => root, + Err(Error::NotFound(_)) => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut paths = vec![backend_prefix.clone()]; + if root.is_dir { + paths.extend( + self.primary + .tree_directory( + &backend_prefix, + true, + None, + None, + None, + None, + None, + false, + ) + .await? + .into_iter() + .map(|entry| entry.path), + ); + } + Ok(paths) + }) + .await?; + let mut selected = Vec::new(); + for path in paths { + let logical = format!("{MULTIWRITE_MOUNT_PREFIX}{path}"); + if !is_multiwrite_internal_path(&logical) + && self.router.route(&scope.account_id, &logical).await?.scope == *scope + { + selected.push(logical); + } + } + selected.sort(); + selected.dedup(); + Ok(selected) + } +} + +/// Reads complete validated checkpoint snapshots under the publication lease. +pub struct CheckpointReader { + store: Arc, +} +impl CheckpointReader { + /// Create a reader over the raw metadata store. + pub fn new(store: Arc) -> Self { + Self { store } + } +} + +#[async_trait] +impl CheckpointConsumer for CheckpointReader { + /// Read the latest checkpoint or return a typed retry for immutable races. + async fn read_latest( + &self, + scope: &ScopeKey, + synced_seq: u64, + tail_seq: u64, + ) -> Result { + let pointer_path = self + .store + .paths() + .checkpoints_manifest(&scope.account_id, scope.partition_id)? + .0; + let lease = self + .store + .pathlock_manager() + .acquire_exact( + &pointer_path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(Duration::from_secs(5)), + None, + ) + .await?; + let pointer = async { + let pointer: CheckpointsManifest = self.store.read_json(&pointer_path).await?; + pointer.validate()?; + let Some(latest) = pointer.latest_checkpoint else { + return Ok(None); + }; + if latest.checkpoint_to_seq <= synced_seq || latest.checkpoint_to_seq > tail_seq { + return Ok(None); + } + Ok(Some(latest)) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + let latest = finish_with_release(pointer, release)?; + let Some(latest) = latest else { + return Ok(CheckpointReadResult::NoCheckpoint); + }; + let states = match read_published(self.store.as_ref(), scope, &latest).await { + Ok(states) => states, + Err(Error::NotFound(_)) | Err(Error::Serialization(_)) => { + return Ok(CheckpointReadResult::RetryLatestCheckpoint) + } + Err(error) => return Err(error), + }; + let account_manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0; + let account_manifest: PartitionsManifest = + self.store.read_json(&account_manifest_path).await?; + account_manifest.validate()?; + let mut directory_prefixes = BTreeSet::new(); + for event in account_manifest.directory_events { + if event.positions.iter().any(|position| { + position.partition_id == scope.partition_id + && position.epoch == scope.epoch + && position.seq <= latest.checkpoint_to_seq + }) { + directory_prefixes.insert(event.source_path); + if event.operation == DirectoryOperation::MoveTree { + directory_prefixes.insert(event.destination_path.ok_or_else(|| { + Error::Serialization("move directory event lacks destination".into()) + })?); + } + } + } + Ok(CheckpointReadResult::Snapshot(CheckpointSnapshot { + checkpoint_to_seq: latest.checkpoint_to_seq, + file_states: states, + directory_prefixes: directory_prefixes.into_iter().collect(), + })) + } +} + +/// Encode a deterministic checkpoint tree while continuing the caller's chunk index. +fn encode_states_with_index( + scope: &ScopeKey, + mut states: Vec, + index: &mut usize, +) -> Result<(ChunkDescriptor, Vec<(String, Vec)>)> { + let (node, root_path) = checkpoint_node_from_states(scope, &mut states)?; + encode_checkpoint_tree(node, &root_path, index) +} + +/// Encode states as one or more non-root checkpoint descriptors. +fn encode_child_states_with_index( + scope: &ScopeKey, + mut states: Vec, + start_path: &str, + end_path: Option<&str>, + index: &mut usize, +) -> Result<(Vec, Vec<(String, Vec)>)> { + if states.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + let (mut node, root_path) = checkpoint_node_from_states(scope, &mut states)?; + let mut files = Vec::new(); + let mut descriptors = Vec::new(); + let children = std::mem::take(&mut node.children); + if node.latest_seq.is_some() { + descriptors.push(encode_node(node, &root_path, false, index, &mut files)?); + } + partition_children(children, &root_path, index, &mut files, &mut descriptors)?; + assign_checkpoint_child_ranges(&mut descriptors, start_path, end_path)?; + Ok((descriptors, files)) +} + +/// Build the account-relative checkpoint radix tree from sorted file states. +fn checkpoint_node_from_states( + scope: &ScopeKey, + states: &mut Vec, +) -> Result<(CheckpointNode, String)> { + for state in states.iter() { + state.validate(scope)?; + } + states.sort_by(|left, right| left.path.cmp(&right.path)); + if states.windows(2).any(|pair| pair[0].path == pair[1].path) { + return Err(Error::Serialization( + "checkpoint contains duplicate paths".into(), + )); + } + let root_path = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id); + let mut node = CheckpointNode { + path_fragment: String::new(), + latest_seq: None, + deleted: None, + children: Vec::new(), + }; + for state in states.iter() { + if !is_canonical_account_path(&state.path, &root_path) { + return Err(Error::Serialization( + "checkpoint path crosses account root".into(), + )); + } + let relative = state + .path + .strip_prefix(&root_path) + .ok_or_else(|| Error::Serialization("checkpoint path crosses account root".into()))?; + insert_state(&mut node, relative, state)?; + } + Ok((node, root_path)) +} + +/// Encode one preconstructed radix tree through the production chunk splitter. +fn encode_checkpoint_tree( + mut node: CheckpointNode, + root_path: &str, + index: &mut usize, +) -> Result<(ChunkDescriptor, Vec<(String, Vec)>)> { + node.validate()?; + let mut files = Vec::new(); + let root = if checkpoint_node_count(&node) <= MAX_CHECKPOINT_NODES { + encode_node(node, root_path, true, index, &mut files)? + } else { + let children = std::mem::take(&mut node.children); + let mut descriptor = encode_node(node, root_path, true, index, &mut files)?; + partition_children( + children, + root_path, + index, + &mut files, + &mut descriptor.chunks, + )?; + assign_checkpoint_child_ranges(&mut descriptor.chunks, root_path, None)?; + descriptor + }; + Ok((root, files)) +} + +/// Insert one file state into the account-relative component radix tree. +fn insert_state(root: &mut CheckpointNode, relative: &str, state: &FileState) -> Result<()> { + let mut node = root; + let mut remaining = relative; + while !remaining.is_empty() { + let Some(index) = node + .children + .iter() + .position(|child| common_prefix_len(&child.path_fragment, remaining) > 0) + else { + node.children.push(CheckpointNode { + path_fragment: remaining.to_string(), + latest_seq: Some(state.latest_seq.seq), + deleted: state.deleted, + children: Vec::new(), + }); + node.children + .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment)); + return Ok(()); + }; + let common = common_prefix_len(&node.children[index].path_fragment, remaining); + if common == node.children[index].path_fragment.len() { + remaining = &remaining[common..]; + node = &mut node.children[index]; + } else { + let mut old = node.children.remove(index); + let suffix = old.path_fragment.split_off(common); + let prefix = std::mem::replace(&mut old.path_fragment, suffix); + let mut parent = CheckpointNode { + path_fragment: prefix, + latest_seq: None, + deleted: None, + children: vec![old], + }; + remaining = &remaining[common..]; + if remaining.is_empty() { + parent.latest_seq = Some(state.latest_seq.seq); + parent.deleted = state.deleted; + } else { + parent.children.push(CheckpointNode { + path_fragment: remaining.to_string(), + latest_seq: Some(state.latest_seq.seq), + deleted: state.deleted, + children: Vec::new(), + }); + parent + .children + .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment)); + } + node.children.insert(index, parent); + return Ok(()); + } + } + if node.latest_seq.is_some() { + return Err(Error::Serialization( + "checkpoint contains duplicate paths".into(), + )); + } + node.latest_seq = Some(state.latest_seq.seq); + node.deleted = state.deleted; + Ok(()) +} + +/// Return the longest UTF-8-safe common prefix length. +fn common_prefix_len(left: &str, right: &str) -> usize { + left.char_indices() + .zip(right.char_indices()) + .take_while(|((_, left), (_, right))| left == right) + .last() + .map_or(0, |((index, ch), _)| index + ch.len_utf8()) +} + +/// Split ordered direct child ranges into bounded immutable chunks. +fn partition_children( + children: Vec, + parent_path: &str, + index: &mut usize, + files: &mut Vec<(String, Vec)>, + descriptors: &mut Vec, +) -> Result<()> { + let mut group = Vec::new(); + let mut group_nodes = 1; + for mut child in children { + let child_nodes = checkpoint_node_count(&child); + if child_nodes + 1 > MAX_CHECKPOINT_NODES { + flush_child_group(&mut group, parent_path, index, files, descriptors)?; + let fragment = child.path_fragment.clone(); + let mut grandchildren = std::mem::take(&mut child.children); + if child.latest_seq.is_some() { + descriptors.push(encode_node(child, parent_path, false, index, files)?); + } + for grandchild in &mut grandchildren { + grandchild.path_fragment.insert_str(0, &fragment); + } + partition_children(grandchildren, parent_path, index, files, descriptors)?; + } else { + if group_nodes + child_nodes > MAX_CHECKPOINT_NODES { + flush_child_group(&mut group, parent_path, index, files, descriptors)?; + group_nodes = 1; + } + group_nodes += child_nodes; + group.push(child); + } + } + flush_child_group(&mut group, parent_path, index, files, descriptors) +} + +/// Encode one non-empty direct child range as a synthetic-root chunk. +fn flush_child_group( + children: &mut Vec, + parent_path: &str, + index: &mut usize, + files: &mut Vec<(String, Vec)>, + descriptors: &mut Vec, +) -> Result<()> { + if children.is_empty() { + return Ok(()); + } + let node = CheckpointNode { + path_fragment: String::new(), + latest_seq: None, + deleted: None, + children: std::mem::take(children), + }; + descriptors.push(encode_node(node, parent_path, false, index, files)?); + Ok(()) +} + +/// Encode one bounded radix subtree and derive its exact path range. +fn encode_node( + node: CheckpointNode, + root_path: &str, + root: bool, + index: &mut usize, + files: &mut Vec<(String, Vec)>, +) -> Result { + let (count, first_path) = checkpoint_node_range(&node, root_path); + let start_path = if root { + None + } else { + Some( + first_path + .or_else(|| checkpoint_node_start_path(&node, root_path)) + .ok_or_else(|| { + Error::Serialization("checkpoint child chunk lacks start path".into()) + })?, + ) + }; + let bytes = encode_checkpoint_chunk(&node)?; + let checksum = sha256_hex(&bytes); + let file = checkpoint_chunk_file_name(*index, &checksum); + *index += 1; + files.push((file.clone(), bytes)); + Ok(ChunkDescriptor { + file, + root_path: root_path.to_string(), + start_path, + end_path: None, + file_state_count: count as u32, + checksum, + chunks: Vec::new(), + }) +} + +/// Count nodes in one radix subtree. +fn checkpoint_node_count(root: &CheckpointNode) -> usize { + let mut nodes = vec![root]; + let mut count = 0; + while let Some(node) = nodes.pop() { + count += 1; + nodes.extend(&node.children); + } + count +} + +/// Return the state count and first ordered path encoded in one chunk. +fn checkpoint_node_range(root: &CheckpointNode, root_path: &str) -> (usize, Option) { + let mut paths = Vec::new(); + collect_node_paths(root, root_path, &mut paths); + (paths.len(), paths.first().cloned()) +} + +/// Return the lowest owned path prefix for one encoded checkpoint subtree. +fn checkpoint_node_start_path(node: &CheckpointNode, root_path: &str) -> Option { + let path = format!("{root_path}{}", node.path_fragment); + if node.latest_seq.is_some() || !node.path_fragment.is_empty() { + return Some(path); + } + node.children + .first() + .map(|child| format!("{path}{}", child.path_fragment)) +} + +/// Assign contiguous left-closed and right-open ownership ranges to siblings. +fn assign_checkpoint_child_ranges( + children: &mut Vec, + start_path: &str, + end_path: Option<&str>, +) -> Result<()> { + if children.is_empty() { + return Ok(()); + } + children.sort_by(|left, right| left.start_path.cmp(&right.start_path)); + let starts = children + .iter() + .map(|child| { + child.start_path.clone().ok_or_else(|| { + Error::Serialization("checkpoint child chunk lacks start path".into()) + }) + }) + .collect::>>()?; + for index in 0..children.len() { + children[index].start_path = Some(if index == 0 { + start_path.to_string() + } else { + starts[index].clone() + }); + children[index].end_path = if index + 1 < starts.len() { + Some(starts[index + 1].clone()) + } else { + end_path.map(str::to_string) + }; + } + Ok(()) +} + +/// Collect state paths from one radix subtree in canonical order. +fn collect_node_paths(node: &CheckpointNode, parent: &str, paths: &mut Vec) { + let path = format!("{parent}{}", node.path_fragment); + if node.latest_seq.is_some() { + paths.push(path.clone()); + } + for child in &node.children { + collect_node_paths(child, &path, paths); + } +} + +/// Read and validate one published checkpoint reference. +async fn read_published( + store: &MetadataStore, + scope: &ScopeKey, + latest: &LatestCheckpoint, +) -> Result> { + let manifest = read_manifest(store, scope, &latest.path).await?; + if manifest.checkpoint_to_seq != latest.checkpoint_to_seq { + return Err(Error::Serialization( + "checkpoint pointer sequence mismatch".into(), + )); + } + read_chunks(store, scope, &manifest, &latest.path).await +} + +/// Read and validate one immutable checkpoint manifest. +async fn read_manifest( + store: &MetadataStore, + scope: &ScopeKey, + candidate: &str, +) -> Result { + let directory = checkpoint_directory(store, scope, candidate)?; + let manifest: CheckpointManifest = store + .read_json(&format!("{directory}/{MANIFEST_FILE}")) + .await?; + manifest.validate_for_scope(scope)?; + if tree_checksum(&manifest.root)? != manifest.checksum { + return Err(Error::Serialization( + "checkpoint tree checksum mismatch".into(), + )); + } + Ok(manifest) +} + +/// Build the second-resolution checkpoint directory name used by new publications. +fn checkpoint_candidate_name(created_at_ns: u64) -> Result { + let seconds = i64::try_from(created_at_ns / 1_000_000_000) + .map_err(|_| Error::internal("checkpoint timestamp overflow"))?; + let datetime = DateTime::::from_timestamp(seconds, 0) + .ok_or_else(|| Error::internal("checkpoint timestamp is out of range"))?; + Ok(format!( + "{CHECKPOINT_DIR_PREFIX}{}/", + datetime.format(CHECKPOINT_DIR_TIME_FORMAT) + )) +} + +/// Build a checkpoint chunk filename without embedding any sequence number. +fn checkpoint_chunk_file_name(index: usize, checksum: &str) -> String { + format!("{CHECKPOINT_CHUNK_FILE_PREFIX}{index:06}-{checksum}{CHECKPOINT_CHUNK_FILE_EXTENSION}") +} + +/// Rename descriptor file references while recording old-to-new chunk copies. +fn rename_checkpoint_chunk_files( + descriptor: &mut ChunkDescriptor, + index: &mut usize, + copies: &mut Vec<(String, String, String)>, +) { + let source = descriptor.file.clone(); + let candidate = checkpoint_chunk_file_name(*index, &descriptor.checksum); + *index += 1; + descriptor.file = candidate.clone(); + copies.push((source, candidate, descriptor.checksum.clone())); + for child in &mut descriptor.chunks { + rename_checkpoint_chunk_files(child, index, copies); + } +} + +/// Copy a published checkpoint manifest and referenced chunks into a candidate directory. +async fn copy_checkpoint_files( + store: &MetadataStore, + scope: &ScopeKey, + source: &str, + candidate: &str, + manifest: &CheckpointManifest, +) -> Result { + let source_directory = checkpoint_directory(store, scope, source)?; + let candidate_directory = checkpoint_directory(store, scope, candidate)?; + let mut candidate_manifest = manifest.clone(); + let mut copies = Vec::new(); + let mut index = 0; + rename_checkpoint_chunk_files(&mut candidate_manifest.root, &mut index, &mut copies); + candidate_manifest.checksum = tree_checksum(&candidate_manifest.root)?; + candidate_manifest.validate_for_scope(scope)?; + for (source_file, candidate_file, checksum) in copies { + let bytes = store + .read_bytes(&format!("{source_directory}/{source_file}")) + .await?; + if sha256_hex(&bytes) != checksum { + return Err(Error::Serialization( + "checkpoint chunk checksum mismatch".into(), + )); + } + store + .write_bytes(&format!("{candidate_directory}/{candidate_file}"), &bytes) + .await?; + } + store + .write_bytes( + &format!("{candidate_directory}/{MANIFEST_FILE}"), + &serde_json::to_vec(&candidate_manifest)?, + ) + .await?; + Ok(candidate_manifest) +} + +/// Write checkpoint chunk bytes into one candidate directory. +async fn write_checkpoint_chunks( + store: &MetadataStore, + scope: &ScopeKey, + candidate: &str, + chunks: &[(String, Vec)], +) -> Result<()> { + let directory = checkpoint_directory(store, scope, candidate)?; + for (file, bytes) in chunks { + store + .write_bytes(&format!("{directory}/{file}"), bytes) + .await?; + } + Ok(()) +} + +/// Read and validate one checkpoint chunk without reading its child descriptors. +async fn read_checkpoint_chunk_states( + store: &MetadataStore, + scope: &ScopeKey, + directory: &str, + descriptor: &ChunkDescriptor, + root: bool, +) -> Result> { + let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id); + if descriptor.root_path != account_root { + return Err(Error::Serialization( + "checkpoint descriptor root mismatch".into(), + )); + } + let bytes = store + .read_bytes(&format!("{directory}/{}", descriptor.file)) + .await?; + if sha256_hex(&bytes) != descriptor.checksum { + return Err(Error::Serialization( + "checkpoint chunk checksum mismatch".into(), + )); + } + let node = decode_checkpoint_chunk(&bytes)?; + if encode_checkpoint_chunk(&node)? != bytes { + return Err(Error::Serialization( + "checkpoint chunk is not canonical".into(), + )); + } + let mut states = Vec::new(); + flatten_node(scope, &node, &descriptor.root_path, true, &mut states)?; + if states.len() != descriptor.file_state_count as usize { + return Err(Error::Serialization( + "checkpoint chunk state count mismatch".into(), + )); + } + if states.windows(2).any(|pair| pair[0].path >= pair[1].path) + || states + .iter() + .any(|state| !is_canonical_account_path(&state.path, &account_root)) + { + return Err(Error::Serialization( + "checkpoint chunk paths are invalid or unordered".into(), + )); + } + if !root { + let (start, end) = checkpoint_descriptor_range(descriptor)?; + if states + .iter() + .any(|state| !checkpoint_range_contains(start, end, &state.path)) + { + return Err(Error::Serialization( + "checkpoint chunk range mismatch".into(), + )); + } + } + Ok(states) +} + +/// Merge old chunk states with replacement states that belong to one target range. +fn merge_checkpoint_states( + states: Vec, + delta: &SegmentDelta, + remaining: &mut BTreeMap, + target: ChunkMergeTarget<'_>, +) -> Vec { + let mut merged = states + .into_iter() + .filter(|state| !checkpoint_state_is_dirty(&state.path, delta)) + .map(|state| (state.path.clone(), state)) + .collect::>(); + let selected = remaining + .keys() + .filter(|path| checkpoint_target_contains(&target, path)) + .cloned() + .collect::>(); + for path in selected { + if let Some(state) = remaining.remove(&path) { + merged.insert(path, state); + } + } + merged.into_values().collect() +} + +/// Return whether one merge target owns a logical checkpoint path. +fn checkpoint_target_contains(target: &ChunkMergeTarget<'_>, path: &str) -> bool { + match target { + ChunkMergeTarget::All => true, + ChunkMergeTarget::Range(start, end) => checkpoint_range_contains(start, *end, path), + ChunkMergeTarget::RootOnly(root) => path == *root, + } +} + +/// Return whether a checkpoint state path is replaced or invalidated by a delta. +fn checkpoint_state_is_dirty(path: &str, delta: &SegmentDelta) -> bool { + delta.exact_paths.contains(path) + || delta + .prefixes + .iter() + .any(|prefix| is_path_within(path, prefix)) +} + +/// Return whether a non-root chunk range intersects one segment delta. +fn checkpoint_descriptor_is_dirty( + descriptor: &ChunkDescriptor, + delta: &SegmentDelta, +) -> Result { + let (start, end) = checkpoint_descriptor_range(descriptor)?; + Ok(delta + .exact_paths + .iter() + .any(|path| checkpoint_range_contains(start, end, path)) + || delta + .states + .keys() + .any(|path| checkpoint_range_contains(start, end, path)) + || delta + .prefixes + .iter() + .any(|prefix| checkpoint_range_overlaps_prefix(start, end, prefix))) +} + +/// Return the validated left-closed and right-open range for a non-root descriptor. +fn checkpoint_descriptor_range(descriptor: &ChunkDescriptor) -> Result<(&str, Option<&str>)> { + let start = descriptor + .start_path + .as_deref() + .ok_or_else(|| Error::Serialization("checkpoint child chunk lacks start path".into()))?; + let end = descriptor.end_path.as_deref(); + if let Some(end) = end { + if start >= end { + return Err(Error::Serialization( + "checkpoint chunk range is reversed".into(), + )); + } + } + Ok((start, end)) +} + +/// Return whether a left-closed range contains one logical path. +fn checkpoint_range_contains(start: &str, end: Option<&str>, path: &str) -> bool { + start <= path && end.is_none_or(|end| path < end) +} + +/// Return whether one ownership range can contain paths under a dirty prefix. +fn checkpoint_range_overlaps_prefix(start: &str, end: Option<&str>, prefix: &str) -> bool { + checkpoint_range_contains(start, end, prefix) || is_path_within(start, prefix) +} + +/// Sort child descriptors and normalize contiguous ownership ranges. +fn sort_checkpoint_children(children: &mut Vec, root_path: &str) -> Result<()> { + assign_checkpoint_child_ranges(children, root_path, None) +} + +/// Retain live states and tombstones still needed by at least one backend. +fn retain_checkpoint_states(states: Vec, min_synced: Option) -> Vec { + states + .into_iter() + .filter(|state| { + state.deleted.is_none() + || min_synced.is_some_and(|synced| state.latest_seq.seq >= synced) + }) + .collect() +} + +/// Return every chunk descriptor reachable from a root descriptor. +fn checkpoint_chunk_descriptors(root: &ChunkDescriptor) -> Vec<&ChunkDescriptor> { + let mut descriptors = Vec::new(); + let mut stack = vec![root]; + while let Some(descriptor) = stack.pop() { + descriptors.push(descriptor); + stack.extend(descriptor.chunks.iter()); + } + descriptors +} + +/// Return every non-empty chunk filename reachable from one descriptor tree. +fn checkpoint_chunk_files(root: &ChunkDescriptor) -> Vec { + checkpoint_chunk_descriptors(root) + .into_iter() + .filter_map(|descriptor| (!descriptor.file.is_empty()).then(|| descriptor.file.clone())) + .collect() +} + +/// Count chunk files already referenced by one descriptor tree. +fn checkpoint_chunk_count(root: &ChunkDescriptor) -> usize { + checkpoint_chunk_files(root).len() +} + +/// Best-effort delete candidate chunk files no longer referenced after a rewrite. +async fn remove_replaced_checkpoint_files( + store: &MetadataStore, + scope: &ScopeKey, + candidate: &str, + old_files: Vec, + new_root: &ChunkDescriptor, +) { + let Ok(directory) = checkpoint_directory(store, scope, candidate) else { + return; + }; + let retained = checkpoint_chunk_files(new_root) + .into_iter() + .collect::>(); + for file in old_files { + if retained.contains(&file) { + continue; + } + if let Err(error) = store.remove_file(&format!("{directory}/{file}")).await { + warn!(file = %file, error = %error, "checkpoint chunk cleanup failed"); + } + } +} + +/// Stream-validate a candidate checkpoint without materializing all file states. +async fn verify_checkpoint( + store: &MetadataStore, + scope: &ScopeKey, + manifest: &CheckpointManifest, + candidate: &str, + tail: u64, +) -> Result<()> { + if tree_checksum(&manifest.root)? != manifest.checksum { + return Err(Error::Serialization( + "checkpoint tree checksum mismatch".into(), + )); + } + let directory = checkpoint_directory(store, scope, candidate)?; + let mut descriptors = vec![(&manifest.root, true)]; + let mut previous_path: Option = None; + while let Some((descriptor, root)) = descriptors.pop() { + let states = + read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?; + for state in states { + if state.latest_seq.seq > tail + || previous_path + .as_ref() + .is_some_and(|previous| previous >= &state.path) + { + return Err(Error::Serialization( + "checkpoint candidate verification failed".into(), + )); + } + previous_path = Some(state.path); + } + descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false))); + } + Ok(()) +} + +/// Read every referenced chunk and reconstruct sorted file states. +async fn read_chunks( + store: &MetadataStore, + scope: &ScopeKey, + manifest: &CheckpointManifest, + candidate: &str, +) -> Result> { + let directory = checkpoint_directory(store, scope, candidate)?; + let mut descriptors = vec![(&manifest.root, true)]; + let mut states = Vec::new(); + while let Some((descriptor, root)) = descriptors.pop() { + states.extend( + read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?, + ); + descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false))); + } + if states.windows(2).any(|pair| pair[0].path >= pair[1].path) { + return Err(Error::Serialization( + "checkpoint paths are not strictly ordered".into(), + )); + } + Ok(states) +} + +/// Flatten one decoded checkpoint node into validated file states. +fn flatten_node( + scope: &ScopeKey, + node: &CheckpointNode, + parent: &str, + root: bool, + states: &mut Vec, +) -> Result<()> { + if (!root || !node.path_fragment.is_empty()) && !valid_path_fragment(&node.path_fragment) { + return Err(Error::Serialization( + "checkpoint path fragment is not relative".into(), + )); + } + let path = format!("{parent}{}", node.path_fragment); + if let Some(seq) = node.latest_seq { + let state = FileState { + path: path.clone(), + latest_seq: ScopedSeq { + scope: scope.clone(), + seq, + }, + deleted: node.deleted, + }; + state.validate(scope)?; + states.push(state); + } + for child in &node.children { + flatten_node(scope, child, &path, false, states)?; + } + Ok(()) +} + +/// Resolve one isolated checkpoint directory from its relative latest pointer. +fn checkpoint_directory( + store: &MetadataStore, + scope: &ScopeKey, + candidate: &str, +) -> Result { + let name = candidate.strip_suffix('/').ok_or_else(|| { + Error::Serialization("checkpoint pointer must reference a directory".into()) + })?; + if !is_checkpoint_directory_name(name) { + return Err(Error::Serialization( + "checkpoint pointer directory is invalid".into(), + )); + } + let root = store + .paths() + .checkpoints_dir(&scope.account_id, scope.partition_id)? + .0; + Ok(format!("{root}/{name}")) +} + +/// Return whether one radix edge is a non-empty relative fragment. +fn valid_path_fragment(fragment: &str) -> bool { + !fragment.is_empty() && !fragment.contains('\0') +} + +/// Return whether a path is canonical and belongs to one account root. +fn is_canonical_account_path(path: &str, root: &str) -> bool { + is_path_within(path, root) + && !path.ends_with('/') + && path + .split('/') + .skip(1) + .all(|part| !part.is_empty() && part != "." && part != "..") +} + +/// Hash the canonical serialized descriptor tree. +fn tree_checksum(root: &ChunkDescriptor) -> Result { + Ok(sha256_hex(&serde_json::to_vec(root)?)) +} + +/// Return whether a path equals a prefix or is one of its descendants. +fn is_path_within(path: &str, prefix: &str) -> bool { + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +/// Return the current Unix timestamp in nanoseconds. +fn now_ns() -> Result { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| Error::internal(error.to_string()))? + .as_nanos(); + u64::try_from(nanos).map_err(|_| Error::internal("checkpoint timestamp overflow")) +} + +/// Preserve an operation result while requiring lease release to succeed. +fn finish_with_release(operation: Result, release: Result<()>) -> Result { + match (operation, release) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + } +} diff --git a/crates/ragfs/src/multibackend/codec.rs b/crates/ragfs/src/multibackend/codec.rs new file mode 100644 index 0000000000..bd0e527a5a --- /dev/null +++ b/crates/ragfs/src/multibackend/codec.rs @@ -0,0 +1,359 @@ +//! Stable binary codecs for V2 segment and checkpoint files. + +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::core::errors::{Error, Result}; +use crate::multibackend::constants::{MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS}; +use crate::multibackend::model::{CheckpointNode, DecodedSegment, SegmentEventType, SegmentRecord}; + +const SEGMENT_MAGIC: &[u8; 4] = b"OVSG"; +const CHECKPOINT_MAGIC: &[u8; 4] = b"OVCP"; +const FORMAT_VERSION: u16 = 1; +const MAX_STRING_LEN: usize = 1024 * 1024; +const MIN_RECORD_LEN: usize = 14; +const MAX_RECORD_LEN: usize = 2 * 1024 * 1024; +const MAX_CHECKPOINT_DEPTH: usize = 256; + +/// Encode segment records in the stable OVSG binary format. +pub fn encode_segment(records: &[SegmentRecord]) -> Result> { + if records.len() > MAX_SEGMENT_RECORDS { + return Err(serialization("segment has too many records")); + } + + let mut bytes = Vec::new(); + bytes.extend_from_slice(SEGMENT_MAGIC); + bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); + for record in records { + record.validate()?; + let frame = encode_record(record)?; + if frame.len() > MAX_RECORD_LEN { + return Err(serialization("segment record is too large")); + } + bytes.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&frame); + } + Ok(bytes) +} + +/// Decode complete segment records and recover a truncated final frame. +pub fn decode_segment(bytes: &[u8]) -> Result { + let mut reader = Reader::new(bytes); + read_header(&mut reader, SEGMENT_MAGIC)?; + let mut records = Vec::new(); + let mut valid_bytes = reader.position(); + let mut truncated_tail = false; + + while reader.remaining() > 0 { + if records.len() == MAX_SEGMENT_RECORDS { + return Err(serialization("segment has too many records")); + } + let frame_start = reader.position(); + if reader.remaining() < 4 { + truncated_tail = true; + break; + } + let frame_len = reader.read_u32()? as usize; + if frame_len < MIN_RECORD_LEN { + return Err(serialization("segment record is too small")); + } + if frame_len > MAX_RECORD_LEN { + return Err(serialization("segment record is too large")); + } + if frame_len > reader.remaining() { + truncated_tail = true; + break; + } + let mut frame = Reader::new(reader.read_exact(frame_len)?); + let record = decode_record(&mut frame)?; + if frame.remaining() != 0 { + return Err(serialization("segment record has trailing bytes")); + } + records.push(record); + valid_bytes = reader.position(); + debug_assert!(valid_bytes > frame_start); + } + + Ok(DecodedSegment { + records, + valid_bytes: valid_bytes as u64, + truncated_tail, + }) +} + +/// Encode one checkpoint tree in the stable OVCP binary format. +pub fn encode_checkpoint_chunk(root: &CheckpointNode) -> Result> { + let node_count = validate_checkpoint_shape(root)?; + root.validate()?; + + let mut bytes = Vec::new(); + bytes.extend_from_slice(CHECKPOINT_MAGIC); + bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); + bytes.extend_from_slice(&(node_count as u32).to_le_bytes()); + encode_checkpoint_node(root, &mut bytes)?; + Ok(bytes) +} + +/// Decode one complete OVCP checkpoint tree. +pub fn decode_checkpoint_chunk(bytes: &[u8]) -> Result { + let mut reader = Reader::new(bytes); + read_header(&mut reader, CHECKPOINT_MAGIC)?; + let node_count = reader.read_u32()? as usize; + if node_count == 0 || node_count > MAX_CHECKPOINT_NODES { + return Err(serialization("invalid checkpoint node count")); + } + + let mut remaining_nodes = node_count; + let root = decode_checkpoint_node(&mut reader, &mut remaining_nodes, 1)?; + if remaining_nodes != 0 { + return Err(serialization( + "checkpoint node count does not match payload", + )); + } + if reader.remaining() != 0 { + return Err(serialization("checkpoint has trailing bytes")); + } + root.validate()?; + Ok(root) +} + +/// Return a lowercase hexadecimal SHA-256 digest. +pub fn sha256_hex(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Encode one validated segment record payload. +fn encode_record(record: &SegmentRecord) -> Result> { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&record.seq.to_le_bytes()); + bytes.push(record.event_type as u8); + let flags = + u8::from(record.op_id.is_some()) | (u8::from(record.destination_path.is_some()) << 1); + bytes.push(flags); + write_string(&mut bytes, &record.path)?; + if let Some(op_id) = record.op_id { + bytes.extend_from_slice(op_id.as_bytes()); + } + if let Some(destination) = &record.destination_path { + write_string(&mut bytes, destination)?; + } + Ok(bytes) +} + +/// Decode and validate one complete segment record payload. +fn decode_record(reader: &mut Reader<'_>) -> Result { + let seq = reader.read_u64()?; + let event_type = match reader.read_u8()? { + 1 => SegmentEventType::Write, + 2 => SegmentEventType::Remove, + 3 => SegmentEventType::RemoveTree, + 4 => SegmentEventType::MoveTree, + 5 => SegmentEventType::DeleteScope, + _ => return Err(serialization("unknown segment event type")), + }; + let flags = reader.read_u8()?; + if flags & !0b11 != 0 { + return Err(serialization("unknown segment record flags")); + } + let path = reader.read_string()?; + let op_id = if flags & 1 != 0 { + let raw: [u8; 16] = reader + .read_exact(16)? + .try_into() + .map_err(|_| serialization("invalid operation id"))?; + Some(Uuid::from_bytes(raw)) + } else { + None + }; + let destination_path = if flags & 2 != 0 { + Some(reader.read_string()?) + } else { + None + }; + let record = SegmentRecord { + seq, + event_type, + path, + op_id, + destination_path, + }; + record.validate()?; + Ok(record) +} + +/// Validate checkpoint size and depth without recursive traversal. +fn validate_checkpoint_shape(root: &CheckpointNode) -> Result { + let mut stack = vec![(root, 1_usize)]; + let mut node_count = 0_usize; + while let Some((node, depth)) = stack.pop() { + if depth > MAX_CHECKPOINT_DEPTH { + return Err(serialization("checkpoint exceeds maximum depth")); + } + node_count += 1; + if node_count > MAX_CHECKPOINT_NODES { + return Err(serialization("checkpoint has too many nodes")); + } + if node.path_fragment.len() > MAX_STRING_LEN { + return Err(serialization("checkpoint path fragment is too large")); + } + stack.extend(node.children.iter().map(|child| (child, depth + 1))); + } + Ok(node_count) +} + +/// Encode one checkpoint node after iterative shape validation. +fn encode_checkpoint_node(node: &CheckpointNode, bytes: &mut Vec) -> Result<()> { + write_string(bytes, &node.path_fragment)?; + let flags = u8::from(node.latest_seq.is_some()) | (u8::from(node.deleted.is_some()) << 1); + bytes.push(flags); + if let Some(seq) = node.latest_seq { + bytes.extend_from_slice(&seq.to_le_bytes()); + } + bytes.extend_from_slice(&(node.children.len() as u32).to_le_bytes()); + for child in &node.children { + encode_checkpoint_node(child, bytes)?; + } + Ok(()) +} + +/// Decode one node while enforcing global node and depth budgets before allocation. +fn decode_checkpoint_node( + reader: &mut Reader<'_>, + remaining_nodes: &mut usize, + depth: usize, +) -> Result { + if depth > MAX_CHECKPOINT_DEPTH { + return Err(serialization("checkpoint exceeds maximum depth")); + } + if *remaining_nodes == 0 { + return Err(serialization("checkpoint contains too many nodes")); + } + *remaining_nodes -= 1; + + let path_fragment = reader.read_string()?; + let flags = reader.read_u8()?; + if flags & !0b11 != 0 { + return Err(serialization("unknown checkpoint node flags")); + } + let latest_seq = if flags & 1 != 0 { + Some(reader.read_u64()?) + } else { + None + }; + let deleted = if flags & 2 != 0 { Some(true) } else { None }; + let child_count = reader.read_u32()? as usize; + if child_count > *remaining_nodes { + return Err(serialization( + "checkpoint child count exceeds remaining node budget", + )); + } + + let mut children = Vec::new(); + for _ in 0..child_count { + children.push(decode_checkpoint_node(reader, remaining_nodes, depth + 1)?); + } + Ok(CheckpointNode { + path_fragment, + latest_seq, + deleted, + children, + }) +} + +/// Append one bounded length-prefixed UTF-8 string. +fn write_string(bytes: &mut Vec, value: &str) -> Result<()> { + if value.len() > MAX_STRING_LEN { + return Err(serialization("string is too large")); + } + bytes.extend_from_slice(&(value.len() as u32).to_le_bytes()); + bytes.extend_from_slice(value.as_bytes()); + Ok(()) +} + +/// Read and validate a format magic and version. +fn read_header(reader: &mut Reader<'_>, magic: &[u8; 4]) -> Result<()> { + if reader.read_exact(4)? != magic { + return Err(serialization("invalid binary format magic")); + } + if reader.read_u16()? != FORMAT_VERSION { + return Err(serialization("unsupported binary format version")); + } + Ok(()) +} + +/// Build a serialization error with a concise message. +fn serialization(message: impl Into) -> Error { + Error::Serialization(message.into()) +} + +struct Reader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> Reader<'a> { + /// Create a reader positioned at the start of a byte slice. + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + /// Return the current byte offset. + fn position(&self) -> usize { + self.position + } + + /// Return the number of unread bytes. + fn remaining(&self) -> usize { + self.bytes.len() - self.position + } + + /// Read an exact byte range or reject truncated input. + fn read_exact(&mut self, length: usize) -> Result<&'a [u8]> { + let end = self + .position + .checked_add(length) + .ok_or_else(|| serialization("binary length overflow"))?; + let value = self + .bytes + .get(self.position..end) + .ok_or_else(|| serialization("truncated binary data"))?; + self.position = end; + Ok(value) + } + + /// Read one unsigned byte. + fn read_u8(&mut self) -> Result { + Ok(self.read_exact(1)?[0]) + } + + /// Read one little-endian u16. + fn read_u16(&mut self) -> Result { + Ok(u16::from_le_bytes(self.read_exact(2)?.try_into().unwrap())) + } + + /// Read one little-endian u32. + fn read_u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.read_exact(4)?.try_into().unwrap())) + } + + /// Read one little-endian u64. + fn read_u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.read_exact(8)?.try_into().unwrap())) + } + + /// Read one bounded length-prefixed UTF-8 string. + fn read_string(&mut self) -> Result { + let length = self.read_u32()? as usize; + if length > MAX_STRING_LEN { + return Err(serialization("string is too large")); + } + let raw = self.read_exact(length)?; + let value = + std::str::from_utf8(raw).map_err(|_| serialization("string is not valid UTF-8"))?; + Ok(value.to_owned()) + } +} + +#[cfg(test)] +#[path = "tests/test_codec.rs"] +mod tests; diff --git a/crates/ragfs/src/multibackend/config.rs b/crates/ragfs/src/multibackend/config.rs index be305f73c0..f2d0f581c3 100644 --- a/crates/ragfs/src/multibackend/config.rs +++ b/crates/ragfs/src/multibackend/config.rs @@ -1,12 +1,11 @@ //! Multi-backend config validation and normalization helpers. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use serde_json::Value; use crate::core::errors::{Error, Result}; -use crate::core::multibackend_wrapper::{BackendEntry, SyncMode}; -use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig, RedirectPolicy}; +use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig}; /// Convert one nested backup `params` object into plugin config values. pub fn item_params_to_config_values(value: &Value) -> Result> { @@ -66,62 +65,32 @@ pub fn validate_primary_encryption_flags( Ok(()) } -/// Validate redirect targets against the assembled backup entries. -pub fn validate_redirect_targets( - redirects: &[RedirectPolicy], - backup_entries: &[BackendEntry], -) -> Result<()> { - for policy in redirects { - let targets = match policy { - RedirectPolicy::FileOverSizePolicy { target, .. } => target.as_ref(), - RedirectPolicy::FileExtensionPolicy { target, .. } => target.as_ref(), - }; - let target_names = targets - .ok_or_else(|| Error::config("redirect policy target must not be empty".to_string()))?; - if target_names.is_empty() { +/// Validate V2 backup settings. +pub fn validate_backups_config(bc: &BackendsConfig) -> Result<()> { + if !matches!(bc.provider.as_str(), "filesystem" | "cache") { + return Err(Error::config( + "backups.provider must be 'filesystem' or 'cache'".to_string(), + )); + } + + let mut names = HashSet::new(); + for item in &bc.items { + let name = item.name.trim(); + if name.is_empty() { + return Err(Error::config("backup name must not be empty".to_string())); + } + if name == "primary" { return Err(Error::config( - "redirect policy target must not be empty".to_string(), + "backup backend name 'primary' is reserved".to_string(), )); } - for name in target_names { - if !backup_entries.iter().any(|be| &be.name == name) { - return Err(Error::config(format!( - "redirect target '{}' not found in backup entries", - name - ))); - } + if !names.insert(name) { + return Err(Error::config(format!( + "duplicate backup name '{}'", + item.name + ))); } } - Ok(()) -} -/// Validate that exclude policies do not silently carry redirect targets. -pub fn validate_backup_excludes(bc: &BackendsConfig) -> Result<()> { - for item in &bc.items { - for policy in item.excludes.as_deref().unwrap_or(&[]) { - match policy { - RedirectPolicy::FileOverSizePolicy { target, .. } - | RedirectPolicy::FileExtensionPolicy { target, .. } => { - if target.is_some() { - return Err(Error::config(format!( - "exclude policy for backup '{}' must not contain target", - item.name - ))); - } - } - } - } - } Ok(()) } - -/// Convert config sync settings into the runtime sync mode. -pub fn sync_mode_from_config(bc: &BackendsConfig) -> SyncMode { - match bc.sync_type.as_str() { - "sync" => SyncMode::Sync { - ack_count: bc.write_ack_count.unwrap_or(usize::MAX), - timeout_ms: bc.write_ack_timeout_ms.unwrap_or(0), - }, - _ => SyncMode::Async, - } -} diff --git a/crates/ragfs/src/multibackend/constants.rs b/crates/ragfs/src/multibackend/constants.rs new file mode 100644 index 0000000000..160f71154c --- /dev/null +++ b/crates/ragfs/src/multibackend/constants.rs @@ -0,0 +1,38 @@ +//! Shared constants for V2 multi-write metadata. + +/// Canonical logical mount prefix used by V2 multi-write metadata. +pub(crate) const MULTIWRITE_MOUNT_PREFIX: &str = "/local"; +/// Internal metadata directory name. +pub(crate) const SYSTEM_DIR: &str = "_system"; +/// Mount-level protocol manifest file name. +pub(crate) const MULTIWRITE_PROTOCOL_FILE: &str = ".multiwrite.json"; +/// Account partition directory name. +pub(crate) const PARTITIONS_DIR: &str = "partitions"; +/// Standard manifest file name. +pub(crate) const MANIFEST_FILE: &str = "manifest.json"; +/// Segment blob directory name. +pub(crate) const SEGMENTS_DIR: &str = "segments"; +/// Checkpoint directory name. +pub(crate) const CHECKPOINTS_DIR: &str = "checkpoints"; + +/// Mutable head segment file prefix. +pub(crate) const HEAD_SEGMENT_FILE_PREFIX: &str = "head-"; +/// Immutable sealed segment file prefix. +pub(crate) const SEALED_SEGMENT_FILE_PREFIX: &str = "segment-"; +/// Segment blob file extension. +pub(crate) const SEGMENT_FILE_EXTENSION: &str = ".ovsg"; +/// Checkpoint candidate directory prefix. +pub(crate) const CHECKPOINT_DIR_PREFIX: &str = "cp-"; +/// Checkpoint candidate timestamp format. +pub(crate) const CHECKPOINT_DIR_TIME_FORMAT: &str = "%Y%m%d%H%M%S"; +/// Checkpoint chunk file prefix. +pub(crate) const CHECKPOINT_CHUNK_FILE_PREFIX: &str = "chunk-"; +/// Checkpoint chunk file extension. +pub(crate) const CHECKPOINT_CHUNK_FILE_EXTENSION: &str = ".ovcp"; + +/// Total number of fixed routing buckets. +pub const VBUCKETS: u32 = 65_536; +/// Maximum records allowed in one segment. +pub(crate) const MAX_SEGMENT_RECORDS: usize = 8_192; +/// Maximum file states allowed in one checkpoint chunk. +pub(crate) const MAX_CHECKPOINT_NODES: usize = 500_000; \ No newline at end of file diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rs index 0f657d8500..da9d8d4bc2 100644 --- a/crates/ragfs/src/multibackend/factory.rs +++ b/crates/ragfs/src/multibackend/factory.rs @@ -1,6 +1,6 @@ //! Multi-backend runtime assembly from validated mount configuration. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -13,10 +13,12 @@ use crate::core::multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS}; use crate::core::plugin::ServicePlugin; use crate::core::types::{BackendRole, BackendsConfig, ConfigValue, PluginConfig}; use crate::multibackend::config::{ - item_params_to_config_values, sync_mode_from_config, validate_backup_excludes, - validate_primary_encryption_flags, validate_redirect_targets, + item_params_to_config_values, validate_backups_config, validate_primary_encryption_flags, }; -use crate::multibackend::meta::RelativePathFsContextResolver; +use crate::multibackend::meta::{MetadataStore, RelativePathFsContextResolver}; +#[cfg(feature = "cache")] +use crate::multibackend::provider::CacheProvider; +use crate::multibackend::provider::{FilesystemProvider, MultiWriteProvider}; use crate::multibackend::types::MultiBackendBuildContext; use crate::shape::validate::ensure_backend_shape; @@ -51,14 +53,15 @@ pub async fn init_backend_plugin( Ok(Arc::from(fs)) } -/// Build the multi-backend wrapper from primary and backup configuration. -pub async fn build_multi_write_fs( +/// Build the multi-backend wrapper without starting background work. +pub(crate) async fn build_inactive_multi_write_fs( registry: &Arc>>>, config: &PluginConfig, bc: &BackendsConfig, build_ctx: MultiBackendBuildContext, ) -> Result { let global_encryption_enabled = build_ctx.global_encryption_enabled(); + validate_backups_config(bc)?; validate_primary_encryption_flags(config, global_encryption_enabled)?; let primary_raw = init_backend_plugin(registry, &config.name, &config.params).await?; @@ -92,22 +95,37 @@ pub async fn build_multi_write_fs( primary_raw.clone() }; - let mut backup_entries: Vec = Vec::new(); - let mut seen_names: HashSet = HashSet::new(); - - for item in &bc.items { - if item.name == "primary" { - return Err(Error::config( - "backup backend name 'primary' is reserved".to_string(), - )); - } - if !seen_names.insert(item.name.clone()) { - return Err(Error::config(format!( - "duplicate backup name '{}'", - item.name - ))); + let metadata_store = Arc::new(MetadataStore::new( + primary_backend.clone(), + build_ctx.pathlock_manager.clone(), + &build_ctx.backend_prefix, + )?); + let metadata_provider: Arc = match bc.provider.as_str() { + "filesystem" => Arc::new(FilesystemProvider::new(metadata_store.clone())), + "cache" => { + #[cfg(feature = "cache")] + { + let runtime = build_ctx.cache_runtime.clone().ok_or_else(|| { + Error::config("backups.provider = 'cache' requires a top-level CacheRuntime") + })?; + Arc::new(CacheProvider::new( + metadata_store.clone(), + runtime, + bc.namespace.clone(), + )?) + } + #[cfg(not(feature = "cache"))] + { + return Err(Error::config( + "backups.provider = 'cache' requires the ragfs cache feature", + )); + } } + _ => unreachable!("backup provider validated before construction"), + }; + let mut backup_entries: Vec = Vec::new(); + for item in &bc.items { let backup_params = item_params_to_config_values(&item.params)?; let backup_raw = init_backend_plugin(registry, &item.backend, &backup_params).await?; let backup_encrypted = global_encryption_enabled @@ -160,26 +178,17 @@ pub async fn build_multi_write_fs( role: BackendRole::Backup, backend: backup_backend, raw_backend: None, - operations: item.operations.clone().unwrap_or_default(), - excludes: item.excludes.clone().unwrap_or_default(), }); } - validate_redirect_targets(&config.primary_redirects, &backup_entries)?; - validate_backup_excludes(bc)?; - MultiWriteWrappedFS::builder(primary_backend) .with_primary_raw_backend(primary_raw) + .with_metadata_store(metadata_store) + .with_metadata_provider(metadata_provider) .with_backups(backup_entries) - .with_redirects(config.primary_redirects.clone()) - .sync_mode(sync_mode_from_config(bc)) - .write_concurrency(bc.write_concurrency) - .retry_interval(Duration::from_millis( - bc.retry_interval_ms.unwrap_or(30_000), - )) - .retry_backoff_base_ms(bc.retry_backoff_base_ms.unwrap_or(1_000)) - .max_retry_per_round(bc.retry_max_retries_per_round.unwrap_or(3)) - .quarantine_after_failures(bc.retry_quarantine_after_failures.unwrap_or(9)) + .initial_partitions(bc.initial_partitions) + .checkpoint_interval(Duration::from_secs(bc.checkpoint_interval_secs)) .ctx_resolver(Arc::new(RelativePathFsContextResolver)) - .build() + .build_inactive() + .await } diff --git a/crates/ragfs/src/multibackend/gc.rs b/crates/ragfs/src/multibackend/gc.rs new file mode 100644 index 0000000000..64507ca22a --- /dev/null +++ b/crates/ragfs/src/multibackend/gc.rs @@ -0,0 +1,250 @@ +//! Garbage collection for versioned multi-write metadata. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use crate::core::errors::{Error, Result}; +use crate::multibackend::constants::SEGMENT_FILE_EXTENSION; +use crate::multibackend::meta::{MetadataStore, MultiWriteWorker}; +use crate::multibackend::model::{is_checkpoint_directory_name, CheckpointsManifest, DirectoryEvent, LatestCheckpoint,PartitionState, PartitionsManifest, ProtocolStatus, ScopeKey, SegmentManifest,}; +use crate::multibackend::provider::MultiWriteProvider; + +struct ScopeSnapshot { latest: Option, segments: SegmentManifest} + +/// Collects obsolete segment and checkpoint metadata after publication. +pub struct MetadataGc { + store: Arc, + provider: Arc, +} + +impl MetadataGc { + /// Create a collector over the shared metadata store and segment provider. + pub fn new(store: Arc, provider: Arc) -> Self { + Self { store, provider } + } + + /// Collect one scope while treating concurrent account deletion as complete. + pub async fn collect_scope(&self, scope: &ScopeKey) -> Result<()> { + match self.collect_scope_inner(scope).await { + Err(Error::NotFound(_)) => Ok(()), + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Gc); + Err(error) + } + result => result, + } + } + + /// Apply account safety checks before collecting one Stable scope. + async fn collect_scope_inner(&self, scope: &ScopeKey) -> Result<()> { + if self.store.protocol_status().await? != ProtocolStatus::Stable { + return Ok(()); + } + scope.validate()?; + let account = self.store.update_partitions_manifest(&scope.account_id, + |manifest| Ok(manifest.clone())).await?; + if account.partitions.values().any(|entry| entry.state != PartitionState::Stable) + || account.epoch != scope.epoch + || !account.partitions.contains_key(&scope.partition_id) + { + return Ok(()); + } + let stable = stable_partitions(&account); + let positions_complete = + directory_positions_complete(&account.directory_events, &stable, account.epoch); + let retained = if positions_complete { + self.collect_completed_events(&scope.account_id, &account, &stable).await? + } else { + Some(account.directory_events.clone()) + }; + let Some(retained) = retained else { + return Ok(()); + }; + let allow_segments = + positions_complete && directory_positions_complete(&retained, &stable, account.epoch); + let latest = self.read_latest(scope).await?; + if allow_segments { + self.collect_segments(scope, latest.as_ref(), &retained).await?; + } + self.collect_checkpoints(scope).await + } + + /// Remove completed events only if their account identity and content remain unchanged. + async fn collect_completed_events(&self, account_id: &str, + account: &PartitionsManifest, stable: &[u32]) -> Result>> { + let mut snapshots = BTreeMap::new(); + for &partition_id in stable { + let scope = ScopeKey { account_id: account_id.to_string(), + partition_id, epoch: account.epoch }; + snapshots.insert(partition_id, self.read_scope_snapshot(&scope).await?); + } + let completed = account.directory_events.iter() + .filter(|event| event_is_complete(event, &snapshots)) + .cloned().collect::>(); + let account_id = account_id.to_string(); + let expected_epoch = account.epoch; + let expected_stable = stable.to_vec(); + self.store.update_partitions_manifest(&account_id, move |current| { + if current.epoch != expected_epoch + || stable_partitions(current) != expected_stable + { + return Ok(None); + } + current.directory_events.retain(|event| + !completed.iter().any(|candidate| candidate == event)); + Ok(Some(current.directory_events.clone())) + }).await + } + + /// Read a checkpoint pointer briefly and pair it with segment progress. + async fn read_scope_snapshot(&self, scope: &ScopeKey) -> Result { + let latest = self.read_latest(scope).await?; + let segments = self.provider.read_manifest(scope).await?; + segments.validate()?; + Ok(ScopeSnapshot { latest, segments }) + } + + /// Read the latest checkpoint while holding only its pointer Exact lock. + async fn read_latest(&self, scope: &ScopeKey) -> Result> { + let path = self.store.paths().checkpoints_manifest( + &scope.account_id, scope.partition_id)?.0; + let lease = self.store.pathlock_manager().acquire_exact( + &path, + self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)), + None, + ).await?; + let operation = async { + let pointer: CheckpointsManifest = self.store.read_json(&path).await?; + pointer.validate()?; + Ok(pointer.latest_checkpoint) + }.await; + let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from); + finish_with_release(operation, release) + } + + /// Prune one eligible sealed prefix and remove every unreferenced segment blob. + async fn collect_segments(&self, scope: &ScopeKey, latest: Option<&LatestCheckpoint>, + retained: &[DirectoryEvent]) -> Result<()> { + let manifest = self.provider.read_manifest(scope).await?; + manifest.validate()?; + let removable = manifest.segments.iter().take_while(|descriptor| { + descriptor.segment_to_seq.is_some_and(|to| { + latest.is_some_and(|checkpoint| checkpoint.checkpoint_to_seq >= to) + && manifest.backend_states.values().all(|state| state.synced_seq >= to) + && !retained.iter().any(|event| event.positions.iter().any(|position| { + position.partition_id == scope.partition_id + && position.epoch == scope.epoch + && (descriptor.segment_from_seq..=to).contains(&position.seq) + })) + }) + }).map(|descriptor| descriptor.path.clone()).collect::>(); + if !removable.is_empty() { + for descriptor in self.provider.prune_sealed(scope, &removable).await? { + self.remove_segment(scope, &descriptor.path).await?; + } + } + let directory = self.store.paths().segments_dir( + &scope.account_id, scope.partition_id)?.0; + for entry in self.store.list_directory(&directory).await? { + if !entry.is_dir && entry.name.ends_with(SEGMENT_FILE_EXTENSION) { + self.remove_segment(scope, &entry.name).await?; + } + } + Ok(()) + } + + /// Delete one segment blob while holding its Exact path lock. + async fn remove_segment(&self, scope: &ScopeKey, name: &str) -> Result<()> { + let directory = self.store.paths().segments_dir( + &scope.account_id, scope.partition_id)?.0; + let path = format!("{directory}/{name}"); + let lease = self.store.pathlock_manager().acquire_exact( + &path, + self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)), + None, + ).await?; + let operation = async { + let manifest = self.provider.read_manifest(scope).await?; + if manifest.segments.iter().any(|descriptor| descriptor.path == name) { + return Ok(()); + } + self.store.remove_file(&path).await + }.await; + let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from); + finish_with_release(operation, release) + } + + /// Delete non-latest checkpoint candidates in manifest, chunks, directory order. + async fn collect_checkpoints(&self, scope: &ScopeKey) -> Result<()> { + let directory = self.store.paths().checkpoints_dir( + &scope.account_id, scope.partition_id)?.0; + for entry in self.store.list_directory(&directory).await? { + if !entry.is_dir || !is_checkpoint_directory_name(&entry.name) { + continue; + } + let candidate = format!("{directory}/{}", entry.name); + let lease = self.store.pathlock_manager().acquire_exact( + &candidate, + self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)), + None, + ).await?; + let operation = async { + if self.read_latest(scope).await?.as_ref().is_some_and( + |latest| latest.path.trim_end_matches('/') == entry.name) + { + return Ok(()); + } + self.store.remove_all(&candidate).await + }.await; + let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from); + finish_with_release(operation, release)?; + } + Ok(()) + } +} + +/// Return Stable partition identifiers in deterministic order. +fn stable_partitions(manifest: &PartitionsManifest) -> Vec { + manifest.partitions.iter().filter_map(|(&partition_id, entry)| { + (entry.state == PartitionState::Stable).then_some(partition_id) + }).collect() +} + +/// Return whether every event has one current-epoch position per Stable partition. +fn directory_positions_complete( + events: &[DirectoryEvent], + stable: &[u32], + epoch: u64, +) -> bool { + events.iter().all(|event| stable.iter().all(|partition_id| { + event.positions.iter().any(|position| { + position.partition_id == *partition_id && position.epoch == epoch + }) + })) +} + +/// Return whether every marker is absorbed and applied by every current backend. +fn event_is_complete( + event: &DirectoryEvent, + snapshots: &BTreeMap, +) -> bool { + event.positions.iter().all(|position| snapshots.get(&position.partition_id) + .is_some_and(|snapshot| { + snapshot.latest.as_ref().is_some_and( + |latest| latest.checkpoint_to_seq >= position.seq) + && snapshot.segments.backend_states.values().all( + |state| state.synced_seq >= position.seq) + })) +} + +/// Preserve an operation failure while requiring lock release to succeed. +fn finish_with_release(operation: Result, release: Result<()>) -> Result { + match (operation, release) { + (Ok(value), Ok(())) => Ok(value), + (Ok(_), Err(error)) | (Err(error), Ok(())) => Err(error), + (Err(error), Err(release_error)) => Err(Error::internal(format!( + "{error}; lock release failed: {release_error}" + ))), + } +} diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rs index 777a3cdcf8..6c347df442 100644 --- a/crates/ragfs/src/multibackend/meta.rs +++ b/crates/ragfs/src/multibackend/meta.rs @@ -1,26 +1,26 @@ //! Multi-write metadata management. -//! -//! Provides `MetaStateStore` for serialized read-modify-write of `.redirect.json` and -//! `.sync_log.json` through `primary_backend`, and `FsContextResolver` for recovering -//! `FsContext` from paths in background tasks. -use std::collections::HashMap; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; -use tokio::sync::Mutex; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Serialize}; use crate::core::context::{FsContext, FsContextInner, FS_CTX}; use crate::core::errors::{Error, Result}; use crate::core::filesystem::FileSystem; -use crate::core::types::{RedirectMeta, SyncLogMeta, WriteFlag}; +use crate::core::types::{FileInfo, WriteFlag}; +use crate::lock::PathLockManager; +use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, VBUCKETS}; +use crate::multibackend::model::{ + BackendState, CheckpointsManifest, PartitionEntry, PartitionManifest, PartitionState, + PartitionsManifest, ProtocolState, ProtocolStatus, ScopeKey, SegmentManifest, +}; +use crate::multibackend::router::{build_initial_routes, MultiWritePaths}; /// Trait for resolving `FsContext` from a filesystem path. /// -/// Used by background tasks (retry_loop, backfill, system_sync_retry) that lack a -/// foreground request context. Implementations extract `account_id` from the path -/// (e.g. `/local/{account_id}/...`). +/// Implementations extract `account_id` from the path (e.g. `/local/{account_id}/...`). pub trait FsContextResolver: Send + Sync { /// Recover `FsContext` from a normalized path. /// Returns an error if the path cannot be resolved to a valid context. @@ -34,7 +34,10 @@ impl FsContextResolver for DefaultFsContextResolver { fn resolve(&self, path: &str) -> Result { let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect(); // Path format: /local/{account_id}/... - if parts.len() >= 2 && parts[0] == "local" && !parts[1].is_empty() { + if parts.len() >= 2 + && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/') + && !parts[1].is_empty() + { Ok(Arc::new(FsContextInner::new(parts[1].to_string()))) } else { Err(Error::internal(format!( @@ -61,429 +64,461 @@ impl FsContextResolver for RelativePathFsContextResolver { } } -/// Internal file name for redirect metadata. -pub(crate) const REDIRECT_FILE: &str = ".redirect.json"; -/// Internal file name for sync-log metadata. -pub(crate) const SYNC_LOG_FILE: &str = ".sync_log.json"; -/// Hidden multi-write internal file names (redirect / sync-log metadata). -pub(crate) const MULTIWRITE_INTERNAL_NAMES: &[&str] = &[SYNC_LOG_FILE, REDIRECT_FILE]; -const GLOBAL_STATE_FILE: &str = "/_system/.multiwrite.global.json"; -const GLOBAL_STATE_VERSION: u32 = 1; - -type TrackedLock = Mutex<()>; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -struct GlobalMultiWriteState { - version: u32, - next_seq: u64, +/// Closed worker labels used by the multi-write error counter. +#[derive(Clone, Copy)] +pub(crate) enum MultiWriteWorker { + Flush, + Checkpoint, + CatchUp, + Gc, } -impl Default for GlobalMultiWriteState { - /// Create the default persisted global state for multi-write sequencing. - fn default() -> Self { - Self { - version: GLOBAL_STATE_VERSION, - next_seq: 1, - } - } +/// Primary filesystem store for V2 multi-write metadata. +pub struct MetadataStore { + primary: Arc, + pathlock_manager: Arc, + paths: MultiWritePaths, + worker_errors: [AtomicU64; 4], } -/// Unified metadata store for `.redirect.json` and `.sync_log.json`. -/// -/// All reads and writes go through `primary_backend`, inheriting its encryption -/// configuration. Directory-level locks ensure serialized access to both metadata -/// files within the same directory. -pub struct MetaStateStore { - /// Primary backend (may be encrypted) - primary_backend: Arc, - /// Per-directory locks for serialized read-modify-write - dir_locks: Mutex>>, - /// Dedicated global state lock for next_seq persistence. - global_state_lock: Mutex<()>, - /// Context resolver for background tasks - ctx_resolver: Arc, -} - -impl MetaStateStore { - /// Create a new MetaStateStore. +impl MetadataStore { + /// Create a V2 metadata store for one logical mount prefix. pub fn new( - primary_backend: Arc, - ctx_resolver: Arc, - ) -> Self { - Self::with_cache_config(primary_backend, ctx_resolver, 0, Duration::ZERO) + primary: Arc, + pathlock_manager: Arc, + mount_prefix: &str, + ) -> Result { + Ok(Self { + primary, + pathlock_manager, + paths: MultiWritePaths::new(mount_prefix)?, + worker_errors: std::array::from_fn(|_| AtomicU64::new(0)), + }) } - /// Create a new MetaStateStore; cache parameters are retained as no-op compatibility knobs. - pub fn with_cache_config( - primary_backend: Arc, - ctx_resolver: Arc, - _meta_cache_capacity: usize, - _meta_cache_ttl: Duration, - ) -> Self { - Self { - primary_backend, - dir_locks: Mutex::new(HashMap::new()), - global_state_lock: Mutex::new(()), - ctx_resolver, - } + /// Record one final failure for the selected worker. + pub(crate) fn record_worker_error(&self, worker: MultiWriteWorker) { + self.worker_errors[worker as usize].fetch_add(1, Ordering::Relaxed); } - /// Get or create a per-directory lock. - async fn get_dir_lock(&self, dir: &str) -> Arc { - let mut locks = self.dir_locks.lock().await; - if let Some(lock) = locks.get(dir).cloned() { - return lock; - } - let lock = Arc::new(TrackedLock::new(())); - locks.insert(dir.to_string(), lock.clone()); - lock + /// Return the final failure count for the selected worker. + pub(crate) fn worker_error_count(&self, worker: MultiWriteWorker) -> u64 { + self.worker_errors[worker as usize].load(Ordering::Relaxed) } - /// Return the dedicated `_system` context used by global metadata. - fn system_ctx() -> FsContext { - Arc::new(FsContextInner::new("_system".to_string())) + /// Return the logical-to-backend path mapper owned by this store. + pub fn paths(&self) -> &MultiWritePaths { + &self.paths } - /// Build the effective context for metadata access. - fn effective_meta_ctx(dir: &str, ctx: &FsContext) -> FsContext { - if dir == "/" { - Arc::new(Self::system_ctx().with_auto_pathlock_disabled()) - } else { - Arc::new(ctx.with_auto_pathlock_disabled()) + /// List initialized scope directories currently visible on the primary filesystem. + pub(crate) async fn initialized_accounts(&self) -> Result> { + let mut accounts = Vec::new(); + for entry in self.primary.read_internal_dir("/").await? { + if !entry.is_dir { + continue; + } + let manifest = self.paths.account_manifest(&entry.name)?.1; + match self.primary.stat(&manifest).await { + Ok(_) => accounts.push(entry.name), + Err(Error::NotFound(_)) => {} + Err(error) => return Err(error), + } + } + accounts.sort(); + Ok(accounts) + } + + /// Return whether the primary account root still exists. + pub(crate) async fn account_exists(&self, account_id: &str) -> Result { + let root = self + .paths + .raw_backend_path(&format!("{MULTIWRITE_MOUNT_PREFIX}/{account_id}"))?; + match self.primary.stat(&root).await { + Ok(_) => Ok(true), + Err(Error::NotFound(_)) => Ok(false), + Err(error) => Err(error), } } - /// Build the full path for a metadata file in a directory. - fn meta_path(dir: &str, filename: &str) -> String { - if dir == "/" { - format!("/{}", filename) - } else { - format!("{}/{}", dir, filename) - } + /// Return the path lock manager used by V2 metadata operations. + pub(crate) fn pathlock_manager(&self) -> &PathLockManager { + &self.pathlock_manager } - /// Read one JSON metadata file, returning default for missing or empty files. - async fn read_meta(&self, dir: &str, filename: &str, ctx: &FsContext) -> Result - where - T: DeserializeOwned + Default, - { - let path = Self::meta_path(dir, filename); - let effective_ctx = Self::effective_meta_ctx(dir, ctx); - match FS_CTX - .scope(effective_ctx, async { - self.primary_backend.read(&path, 0, 0).await - }) - .await + /// Build the filesystem context used for one logical metadata path. + fn metadata_context(&self, logical_path: &str) -> Result { + let parts: Vec<&str> = logical_path.trim_start_matches('/').split('/').collect(); + if parts.len() >= 2 + && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/') + && !parts[1].is_empty() { - Ok(data) => { - if data.is_empty() { - Ok(T::default()) - } else { - serde_json::from_slice(&data).map_err(Error::from) - } - } - Err(Error::NotFound(_)) => Ok(T::default()), - Err(e) => Err(e), + return Ok(Arc::new( + FsContextInner::new(parts[1].to_string()) + .with_bypass_cache(true) + .with_auto_pathlock_disabled(), + )); } + Err(Error::internal(format!( + "cannot resolve metadata FsContext from path: {logical_path}" + ))) } - /// Read redirect metadata from a directory (returns default if not found). - async fn read_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result { - self.read_meta(dir, REDIRECT_FILE, ctx).await - } - - /// Read sync log metadata from a directory (returns default if not found). - async fn read_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result { - self.read_meta(dir, SYNC_LOG_FILE, ctx).await - } - - /// Read both metadata files from the primary backend. - async fn read_dir_meta_pair( - &self, - dir: &str, - ctx: &FsContext, - ) -> Result<(RedirectMeta, SyncLogMeta)> { - let redirect = self.read_redirect_meta(dir, ctx).await?; - let sync_log = self.read_sync_log_meta(dir, ctx).await?; - Ok((redirect, sync_log)) - } - - /// Write one JSON metadata file to a directory. - async fn write_meta( - &self, - dir: &str, - filename: &str, - meta: &T, - ctx: &FsContext, - ) -> Result<()> - where - T: Serialize, - { - let path = Self::meta_path(dir, filename); - let data = serde_json::to_vec(meta)?; - let effective_ctx = Self::effective_meta_ctx(dir, ctx); + /// Read one complete binary value from a canonical logical path. + pub(crate) async fn read_bytes(&self, logical_path: &str) -> Result> { + let backend_path = self.paths.raw_backend_path(logical_path)?; + let context = self.metadata_context(logical_path)?; FS_CTX - .scope(effective_ctx, async { - self.primary_backend - .write(&path, &data, 0, WriteFlag::Create) - .await - .map(|_| ()) - }) + .scope(context, self.primary.read(&backend_path, 0, 0)) .await } - /// Write redirect metadata to a directory. - async fn write_redirect_meta( - &self, - dir: &str, - meta: &RedirectMeta, - ctx: &FsContext, - ) -> Result<()> { - self.write_meta(dir, REDIRECT_FILE, meta, ctx).await + /// Read and validate the current V2 protocol status. + pub(crate) async fn protocol_status(&self) -> Result { + let state: ProtocolState = self.read_json(&self.paths.mount_protocol().0).await?; + state.validate()?; + Ok(state.status) } - /// Write sync log metadata to a directory. - async fn write_sync_log_meta( + /// Replace protocol bytes under an Exact lock when the current value matches. + pub(crate) async fn compare_protocol( &self, - dir: &str, - meta: &SyncLogMeta, - ctx: &FsContext, - ) -> Result<()> { - self.write_meta(dir, SYNC_LOG_FILE, meta, ctx).await + expected: &[u8], + replacement: &[u8], + ) -> Result { + let logical_path = self.paths.mount_protocol().0; + let lease = self + .pathlock_manager + .acquire_exact( + &logical_path, + self.pathlock_manager + .default_lock_timeout() + .max(std::time::Duration::from_secs(5)), + None, + ) + .await?; + let operation = async { + match self.read_bytes(&logical_path).await { + Ok(current) if current == expected => { + self.write_bytes(&logical_path, replacement).await?; + Ok(true) + } + Ok(_) | Err(Error::NotFound(_)) => Ok(false), + Err(error) => Err(error), + } + } + .await; + let release = self + .pathlock_manager + .release(&lease) + .await + .map_err(Error::from); + merge_operation_and_cleanup(operation, release) } - /// Serialized read-modify-write of both `.redirect.json` and `.sync_log.json` in a directory. - /// - /// Acquires the directory lock, reads both metadata files, applies `op`, and writes both back. - /// This prevents concurrent updates from losing entries. - pub async fn update_dir_meta(&self, dir: &str, ctx: &FsContext, op: F) -> Result<()> + /// Read and deserialize one JSON value from a canonical logical path. + pub async fn read_json(&self, logical_path: &str) -> Result where - F: FnOnce(&mut RedirectMeta, &mut SyncLogMeta) -> Result<()>, + T: DeserializeOwned, { - let lock = self.get_dir_lock(dir).await; - let _guard = lock.lock().await; - - let (mut redirect_meta, mut sync_log_meta) = self.read_dir_meta_pair(dir, ctx).await?; - let original_redirect = redirect_meta.clone(); - let original_sync_log = sync_log_meta.clone(); - - op(&mut redirect_meta, &mut sync_log_meta)?; + let bytes = self.read_bytes(logical_path).await?; + serde_json::from_slice(&bytes).map_err(Error::from) + } - if redirect_meta != original_redirect { - self.write_redirect_meta(dir, &redirect_meta, ctx).await?; - } - if sync_log_meta != original_sync_log { - self.write_sync_log_meta(dir, &sync_log_meta, ctx).await?; - } - Ok(()) + /// Write complete binary bytes to a canonical logical path. + pub(crate) async fn write_bytes(&self, logical_path: &str, bytes: &[u8]) -> Result<()> { + let backend_path = self.paths.raw_backend_path(logical_path)?; + let context = self.metadata_context(logical_path)?; + FS_CTX + .scope(context, async { + self.primary + .ensure_parent_dirs(&backend_path, 0o755) + .await?; + self.primary + .write(&backend_path, bytes, 0, WriteFlag::Create) + .await?; + Ok(()) + }) + .await } - /// Serialized read-modify-write of two directories' metadata (for cross-directory rename). - /// - /// Acquires both directory locks in lexicographic order to prevent deadlock, - /// then reads and updates all four metadata files within the same critical section. - /// Caller must ensure source_dir != target_dir; use update_dir_meta for same-directory case. - pub async fn update_dual_dir_meta( - &self, - source_dir: &str, - target_dir: &str, - ctx: &FsContext, - op: F, - ) -> Result<()> + /// Validate and write one JSON value at a logical path. + pub async fn publish_json(&self, logical_path: &str, value: &T, validate: F) -> Result<()> where - F: FnOnce( - &mut RedirectMeta, - &mut SyncLogMeta, - &mut RedirectMeta, - &mut SyncLogMeta, - ) -> Result<()>, + T: Serialize, + F: FnOnce(&T) -> Result<()>, { - // Acquire locks in lexicographic order to avoid deadlock. - let (first_dir, second_dir) = if source_dir < target_dir { - (source_dir, target_dir) - } else { - (target_dir, source_dir) - }; - - let lock1 = self.get_dir_lock(first_dir).await; - let lock2 = self.get_dir_lock(second_dir).await; - let _guard1 = lock1.lock().await; - let _guard2 = lock2.lock().await; - - let (mut src_redirect, mut src_sync_log) = self.read_dir_meta_pair(source_dir, ctx).await?; - let (mut tgt_redirect, mut tgt_sync_log) = self.read_dir_meta_pair(target_dir, ctx).await?; - let original_src_redirect = src_redirect.clone(); - let original_src_sync_log = src_sync_log.clone(); - let original_tgt_redirect = tgt_redirect.clone(); - let original_tgt_sync_log = tgt_sync_log.clone(); - - op( - &mut src_redirect, - &mut src_sync_log, - &mut tgt_redirect, - &mut tgt_sync_log, - )?; - - if src_redirect != original_src_redirect { - self.write_redirect_meta(source_dir, &src_redirect, ctx) - .await?; - } - if src_sync_log != original_src_sync_log { - self.write_sync_log_meta(source_dir, &src_sync_log, ctx) - .await?; - } - if tgt_redirect != original_tgt_redirect { - self.write_redirect_meta(target_dir, &tgt_redirect, ctx) - .await?; + let bytes = serde_json::to_vec(value)?; + validate(value)?; + self.write_bytes(logical_path, &bytes).await + } + + /// Create one immutable blob or accept an identical existing value. + pub async fn write_immutable(&self, logical_path: &str, bytes: &[u8]) -> Result<()> { + match self.read_bytes(logical_path).await { + Ok(existing) if existing == bytes => Ok(()), + Ok(_) => Err(Error::AlreadyExists(format!( + "immutable metadata differs: {logical_path}" + ))), + Err(Error::NotFound(_)) => self.write_bytes(logical_path, bytes).await, + Err(error) => Err(error), } - if tgt_sync_log != original_tgt_sync_log { - self.write_sync_log_meta(target_dir, &tgt_sync_log, ctx) - .await?; - } - Ok(()) } - /// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries). - pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result { - self.read_redirect_meta(dir, ctx).await + /// List one raw metadata directory without public hidden-name filtering. + pub(crate) async fn list_directory(&self, logical_path: &str) -> Result> { + let backend_path = self.paths.raw_backend_path(logical_path)?; + self.primary.read_internal_dir(&backend_path).await } - /// Read sync log metadata for a directory (public, used by retry_loop). - pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result { - self.read_sync_log_meta(dir, ctx).await + /// Remove one raw metadata file while treating absence as success. + pub(crate) async fn remove_file(&self, logical_path: &str) -> Result<()> { + let backend_path = self.paths.raw_backend_path(logical_path)?; + self.remove_if_present(&backend_path).await } - /// Get a reference to the context resolver. - pub fn ctx_resolver(&self) -> &Arc { - &self.ctx_resolver + /// Remove one raw metadata tree while treating absence as success. + pub(crate) async fn remove_all(&self, logical_path: &str) -> Result<()> { + let backend_path = self.paths.raw_backend_path(logical_path)?; + match self.primary.remove_all(&backend_path).await { + Ok(()) | Err(Error::NotFound(_)) => Ok(()), + Err(error) => Err(error), + } } - /// Get a reference to the primary backend. - pub fn primary_backend(&self) -> &Arc { - &self.primary_backend + /// Recheck destructive GC safety while the caller holds the account Exact lock. + pub(crate) async fn gc_scope_is_safe(&self, scope: &ScopeKey) -> Result { + let path = self.paths.account_manifest(&scope.account_id)?.0; + let manifest: PartitionsManifest = self.read_json(&path).await?; + manifest.validate()?; + scope.validate_against_manifest(&manifest)?; + let stable = manifest + .partitions + .iter() + .filter_map(|(&id, entry)| (entry.state == PartitionState::Stable).then_some(id)) + .collect::>(); + Ok(stable.len() == manifest.partitions.len() + && manifest.directory_events.iter().all(|event| { + stable.iter().all(|id| { + event.positions.iter().any(|position| { + position.partition_id == *id && position.epoch == manifest.epoch + }) + }) + })) + } + + /// Initialize one account under an Exact manifest lock and return its manifest. + pub async fn initialize_account( + &self, + account_id: &str, + partition_count: u32, + backup_names: &[String], + ) -> Result { + let logical_manifest = self.paths.account_manifest(account_id)?.0; + let lease = self + .pathlock_manager + .acquire_exact( + &logical_manifest, + self.pathlock_manager + .default_lock_timeout() + .max(std::time::Duration::from_secs(5)), + None, + ) + .await?; + let result = self + .initialize_account_locked(account_id, partition_count, backup_names) + .await; + let release = self + .pathlock_manager + .release(&lease) + .await + .map_err(Error::from); + merge_operation_and_cleanup(result, release) } - /// Allocate and persist the next global sequence number. - pub async fn next_seq(&self) -> Result { - let _guard = self.global_state_lock.lock().await; - let mut state = self.read_global_state().await?; - let seq = state.next_seq; - state.next_seq = state.next_seq.saturating_add(1); - self.write_global_state(&state).await?; - Ok(seq) + /// Update one partitions manifest while holding its Exact lock. + pub(crate) async fn update_partitions_manifest( + &self, + account_id: &str, + update: F, + ) -> Result + where + F: FnOnce(&mut PartitionsManifest) -> Result, + { + let logical_manifest = self.paths.account_manifest(account_id)?.0; + let lease = self + .pathlock_manager + .acquire_exact( + &logical_manifest, + self.pathlock_manager + .default_lock_timeout() + .max(std::time::Duration::from_secs(5)), + None, + ) + .await?; + let operation = async { + let mut manifest: PartitionsManifest = self.read_json(&logical_manifest).await?; + manifest.validate()?; + let original = manifest.clone(); + let output = update(&mut manifest)?; + validate_partitions_manifest_update(&original, &manifest)?; + manifest.validate()?; + if manifest != original { + self.publish_json(&logical_manifest, &manifest, PartitionsManifest::validate) + .await?; + } + Ok(output) + } + .await; + let release = self + .pathlock_manager + .release(&lease) + .await + .map_err(Error::from); + merge_operation_and_cleanup(operation, release) } - /// Read the persisted global state file. - async fn read_global_state(&self) -> Result { - let ctx = Self::system_ctx(); - match FS_CTX - .scope(ctx.clone(), async { - self.primary_backend.read(GLOBAL_STATE_FILE, 0, 0).await - }) + /// Initialize account metadata while the caller holds the manifest lock. + async fn initialize_account_locked( + &self, + account_id: &str, + partition_count: u32, + backup_names: &[String], + ) -> Result { + let logical_manifest = self.paths.account_manifest(account_id)?.0; + match self + .read_json::(&logical_manifest) .await { - Ok(data) => { - if data.is_empty() { - Ok(GlobalMultiWriteState::default()) - } else { - let state: GlobalMultiWriteState = serde_json::from_slice(&data)?; - if state.version != GLOBAL_STATE_VERSION { - return Err(Error::config(format!( - "unsupported multi-write global state version {}", - state.version - ))); - } - Ok(state) - } + Ok(manifest) => { + manifest.validate()?; + return Ok(manifest); } - Err(Error::NotFound(_)) => Ok(GlobalMultiWriteState::default()), - Err(e) => Err(e), + Err(Error::NotFound(_)) => {} + Err(error) => return Err(error), } - } - /// Persist the global sequence state through the primary backend. - async fn write_global_state(&self, state: &GlobalMultiWriteState) -> Result<()> { - let ctx = Self::system_ctx(); - let data = serde_json::to_vec(state)?; - FS_CTX - .scope(ctx.clone(), async { - self.primary_backend - .ensure_parent_dirs(GLOBAL_STATE_FILE, 0o755) - .await?; - self.primary_backend - .write(GLOBAL_STATE_FILE, &data, 0, WriteFlag::Create) - .await - .map(|_| ()) + let routes = build_initial_routes(partition_count)?; + let partitions = (0..partition_count) + .map(|partition_id| { + ( + partition_id, + PartitionEntry { + state: PartitionState::Stable, + }, + ) }) - .await - } -} - -/// Per-path serialization queue for async write ordering. -/// -/// Ensures that multiple writes to the same path are executed in FIFO order -/// on backup backends, preventing out-of-order application. -pub struct PathSerializer { - queues: Mutex>>, -} - -impl PathSerializer { - /// Create a new PathSerializer. - pub fn new() -> Self { - Self::with_limits(0, Duration::ZERO) - } - - /// Create a PathSerializer; limit arguments are ignored for compatibility. - pub fn with_limits(_capacity: usize, _ttl: Duration) -> Self { - Self { - queues: Mutex::new(HashMap::new()), + .collect::>(); + let manifest = PartitionsManifest { + version: 1, + epoch: 1, + vbuckets: VBUCKETS, + partitions, + routes, + directory_events: Vec::new(), + }; + manifest.validate()?; + + let backend_states = backup_names + .iter() + .map(|name| (name.clone(), BackendState { synced_seq: 0 })) + .collect::>(); + for partition_id in 0..partition_count { + let partition = PartitionManifest { + version: 1, + partition_id, + epoch: 1, + }; + self.write_validated_json_immutable( + &self.paths.partition_manifest(account_id, partition_id)?.0, + &partition, + PartitionManifest::validate, + ) + .await?; + + let segments = SegmentManifest { + version: 1, + next_seq: 1, + segments: Vec::new(), + backend_states: backend_states.clone(), + }; + self.write_validated_json_immutable( + &self.paths.segment_manifest(account_id, partition_id)?.0, + &segments, + SegmentManifest::validate, + ) + .await?; + + let checkpoints = CheckpointsManifest { + version: 1, + latest_checkpoint: None, + }; + self.write_validated_json_immutable( + &self.paths.checkpoints_manifest(account_id, partition_id)?.0, + &checkpoints, + CheckpointsManifest::validate, + ) + .await?; } + self.write_validated_json_immutable( + &logical_manifest, + &manifest, + PartitionsManifest::validate, + ) + .await?; + Ok(manifest) } - /// Run one async operation under the per-path serialization lock. - pub async fn with_path_lock(&self, path: &str, op: F) -> T + /// Validate, serialize, and create one immutable JSON metadata file. + async fn write_validated_json_immutable( + &self, + logical_path: &str, + value: &T, + validate: F, + ) -> Result<()> where - F: FnOnce() -> Fut, - Fut: std::future::Future, + T: Serialize, + F: FnOnce(&T) -> Result<()>, { - let mut queues = self.queues.lock().await; - let lock = queues - .entry(path.to_string()) - .or_insert_with(|| Arc::new(TrackedLock::new(()))) - .clone(); - drop(queues); - - let _guard = lock.lock().await; - op().await + let bytes = serde_json::to_vec(value)?; + validate(value)?; + self.write_immutable(logical_path, &bytes).await } - /// Return the number of tracked queue entries. - #[cfg(test)] - pub async fn len(&self) -> usize { - self.queues.lock().await.len() - } -} - -impl Default for PathSerializer { - fn default() -> Self { - Self::new() + /// Remove a temporary file while treating an absent file as clean. + async fn remove_if_present(&self, backend_path: &str) -> Result<()> { + match self.primary.remove(backend_path).await { + Ok(()) | Err(Error::NotFound(_)) => Ok(()), + Err(error) => Err(error), + } } } -/// Extract the directory path from a file path. -pub(crate) fn parent_dir(path: &str) -> String { - match path.rfind('/') { - Some(0) => "/".to_string(), - Some(pos) => path[..pos].to_string(), - None => "/".to_string(), +/// Reject updates that change the fixed partition routing identity. +fn validate_partitions_manifest_update( + original: &PartitionsManifest, + updated: &PartitionsManifest, +) -> Result<()> { + if original.epoch != updated.epoch + || original.partitions != updated.partitions + || original.routes != updated.routes + { + return Err(Error::invalid_operation( + "partition routing cannot change after account initialization", + )); } + Ok(()) } -/// Extract the file name from a path. -pub(crate) fn file_name(path: &str) -> &str { - match path.rfind('/') { - Some(pos) => &path[pos + 1..], - None => path, +/// Preserve an operation error while requiring cleanup to succeed. +fn merge_operation_and_cleanup(operation: Result, cleanup: Result<()>) -> Result { + match (operation, cleanup) { + (Ok(value), Ok(())) => Ok(value), + (Ok(_), Err(error)) => Err(error), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(cleanup_error)) => Err(Error::internal(format!( + "{error}; cleanup failed: {cleanup_error}" + ))), } } @@ -497,21 +532,6 @@ pub fn current_required_ctx() -> Result { #[cfg(test)] mod tests { use super::*; - use crate::plugins::memfs::MemFileSystem; - use std::sync::Arc; - - #[test] - fn test_parent_dir() { - assert_eq!(parent_dir("/a/b/c.txt"), "/a/b"); - assert_eq!(parent_dir("/a"), "/"); - assert_eq!(parent_dir("/"), "/"); - } - - #[test] - fn test_file_name() { - assert_eq!(file_name("/a/b/c.txt"), "c.txt"); - assert_eq!(file_name("/a"), "a"); - } #[test] fn test_default_resolver() { @@ -527,48 +547,4 @@ mod tests { let resolver = DefaultFsContextResolver; assert!(resolver.resolve("/invalid/path").is_err()); } - - - #[tokio::test] - async fn test_invalid_sync_log_json_returns_error() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let store = MetaStateStore::new(primary.clone(), Arc::new(DefaultFsContextResolver)); - let ctx = Arc::new(FsContextInner::new("acct".to_string())); - - FS_CTX - .scope(ctx.clone(), async { - primary - .ensure_parent_dirs("/local/acct/docs/.sync_log.json", 0o755) - .await?; - primary - .write( - "/local/acct/docs/.sync_log.json", - b"{not valid json", - 0, - WriteFlag::Create, - ) - .await?; - Ok::<(), Error>(()) - }) - .await - .unwrap(); - - let result = store.get_sync_log_meta("/local/acct/docs", &ctx).await; - assert!(result.is_err(), "corrupted metadata must fail fast"); - } - - #[tokio::test] - async fn test_next_seq_uses_dedicated_global_lock() { - let primary: Arc = Arc::new(MemFileSystem::new()); - let store = MetaStateStore::new(primary, Arc::new(DefaultFsContextResolver)); - - assert_eq!(store.next_seq().await.unwrap(), 1); - assert_eq!(store.next_seq().await.unwrap(), 2); - - let dir_locks = store.dir_locks.lock().await; - assert!( - !dir_locks.contains_key(GLOBAL_STATE_FILE), - "global sequence locking should not occupy the directory lock pool" - ); - } } diff --git a/crates/ragfs/src/multibackend/migration/mod.rs b/crates/ragfs/src/multibackend/migration/mod.rs new file mode 100644 index 0000000000..2db469c397 --- /dev/null +++ b/crates/ragfs/src/multibackend/migration/mod.rs @@ -0,0 +1,231 @@ +//! V2 protocol initialization and resumable full-data import. + +use std::sync::Arc; + +use uuid::Uuid; + +use crate::core::context::{FsContextInner, FS_CTX}; +use crate::core::errors::{Error, Result}; +use crate::core::filesystem::FileSystem; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR}; +use crate::multibackend::meta::MetadataStore; +use crate::multibackend::model::{ + PendingEvent, PendingEventKind, ProtocolState, ProtocolStatus, ProtocolVersion, + SegmentEventType, +}; +use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider}; +use crate::multibackend::runtime::read_active_accounts; + +/// Reads or initializes the persisted V2 multi-write protocol state. +pub struct ProtocolDetector { + store: Arc, +} + +impl ProtocolDetector { + /// Create a detector backed by the supplied raw-primary metadata store. + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Read the protocol state, creating V2 migrating state when absent. + pub async fn detect(&self) -> Result { + let (protocol_path, _) = self.store.paths().mount_protocol(); + match self.store.read_bytes(&protocol_path).await { + Ok(bytes) => decode_protocol(&bytes), + Err(Error::NotFound(_)) => { + let state = ProtocolState { + protocol_version: ProtocolVersion::V2, + status: ProtocolStatus::Migrating, + }; + let bytes = serde_json::to_vec(&state)?; + match self.store.write_immutable(&protocol_path, &bytes).await { + Ok(()) => Ok(state), + Err(Error::AlreadyExists(_)) => { + decode_protocol(&self.store.read_bytes(&protocol_path).await?) + } + Err(error) => Err(error), + } + } + Err(error) => Err(error), + } + } +} + +/// Imports current primary files into V2 segment metadata. +pub struct FullDataImporter { + primary: Arc, + store: Arc, + provider: Arc, + initial_partitions: u32, + backup_names: Vec, +} + +impl FullDataImporter { + /// Create an importer for one primary, metadata store, and persistence provider. + pub fn new( + primary: Arc, + store: Arc, + provider: Arc, + initial_partitions: u32, + backup_names: Vec, + ) -> Self { + Self { + primary, + store, + provider, + initial_partitions, + backup_names, + } + } + + /// Import every current account file and atomically mark V2 stable. + pub async fn run(&self) -> Result<()> { + let (protocol_path, _) = self.store.paths().mount_protocol(); + let migrating_bytes = self.store.read_bytes(&protocol_path).await?; + if decode_protocol(&migrating_bytes)?.status == ProtocolStatus::Stable { + return Ok(()); + } + + // ponytail: fixed delay assumes account bootstrap finishes within 10s; use readiness signaling if that changes. + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + let mut accounts = read_active_accounts(self.primary.as_ref()) + .await? + .into_iter() + .collect::>(); + accounts.sort(); + self.import_scope(SYSTEM_DIR).await?; + for account in &accounts { + if !read_active_accounts(self.primary.as_ref()) + .await? + .contains(account) + { + continue; + } + self.import_scope(account).await?; + } + self.cleanup_v1_metadata().await?; + + let stable_bytes = serde_json::to_vec(&ProtocolState { + protocol_version: ProtocolVersion::V2, + status: ProtocolStatus::Stable, + })?; + if self + .store + .compare_protocol(&migrating_bytes, &stable_bytes) + .await? + { + return Ok(()); + } + if decode_protocol(&self.store.read_bytes(&protocol_path).await?)?.status + == ProtocolStatus::Stable + { + Ok(()) + } else { + Err(Error::invalid_operation( + "multi-write protocol changed during full-data import", + )) + } + } + + /// Initialize one V2 scope and import its current file states. + async fn import_scope(&self, account: &str) -> Result<()> { + let partitions = self + .store + .initialize_account(account, self.initial_partitions, &self.backup_names) + .await?; + bootstrap_account( + self.provider.as_ref(), + self.store.as_ref(), + account, + &partitions, + ) + .await?; + let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled()); + FS_CTX.scope(context, self.import_account(account)).await + } + + /// Scan one account through the business filesystem. + async fn import_account(&self, account: &str) -> Result<()> { + let root = format!("/{account}"); + let mut pending = vec![root]; + while let Some(directory) = pending.pop() { + let mut entries = self.primary.read_internal_dir(&directory).await?; + entries.sort_by(|left, right| left.name.cmp(&right.name)); + for entry in entries { + let path = child_path(&directory, &entry.name); + let logical_path = format!("{MULTIWRITE_MOUNT_PREFIX}{path}"); + if is_multiwrite_internal_path(&logical_path) { + continue; + } + if entry.is_dir { + pending.push(path); + continue; + } + self.provider + .flush( + None, + vec![PendingEvent { + operation_id: Uuid::new_v4(), + account_id: account.to_string(), + kind: PendingEventKind::Data(SegmentEventType::Write), + path: logical_path, + destination_path: None, + route_hint: None, + }], + ) + .await?; + } + } + Ok(()) + } + + /// Remove V1 metadata through the configured primary filesystem. + /// Returns an error when scanning or primary removal fails. + async fn cleanup_v1_metadata(&self) -> Result<()> { + match self.primary.stat("/_system/.multiwrite.global.json").await { + Ok(_) => {} + Err(Error::NotFound(_)) => return Ok(()), + Err(error) => return Err(error), + } + + let page = self + .primary + .glob_directory("/", "**/.sync_log.json", true, None, None, None) + .await?; + for entry in page.entries { + if entry.is_dir { + continue; + } + match self.primary.remove(&entry.path).await { + Ok(()) | Err(Error::NotFound(_)) => {} + Err(error) => return Err(error), + } + } + + match self + .primary + .remove("/_system/.multiwrite.global.json") + .await + { + Ok(()) | Err(Error::NotFound(_)) => Ok(()), + Err(error) => Err(error), + } + } +} + +/// Deserialize and validate exact V2 protocol bytes. +fn decode_protocol(bytes: &[u8]) -> Result { + let state: ProtocolState = serde_json::from_slice(bytes)?; + state.validate()?; + Ok(state) +} + +/// Join one direct child name to a canonical mount-relative directory. +fn child_path(directory: &str, name: &str) -> String { + if directory == "/" { + format!("/{name}") + } else { + format!("{directory}/{name}") + } +} diff --git a/crates/ragfs/src/multibackend/mod.rs b/crates/ragfs/src/multibackend/mod.rs index 44e8e4ab2c..569a2225a8 100644 --- a/crates/ragfs/src/multibackend/mod.rs +++ b/crates/ragfs/src/multibackend/mod.rs @@ -1,14 +1,32 @@ -//! Multi-backend configuration, factory, metadata, and admin support. +//! Multi-backend configuration, factory, and metadata support. -/// Multi-backend administrative APIs. -pub mod admin; +/// Prefix reconciliation primitives for backup catch-up. +pub mod catch_up; +/// Checkpoint construction, validation, publication, and reading. +pub mod checkpoint; +/// Stable binary codecs for V2 segments and checkpoints. +pub mod codec; /// Multi-backend config validation and normalization helpers. pub mod config; +/// Shared constants for V2 multi-write metadata. +pub(crate) mod constants; /// Multi-backend runtime assembly from validated config. pub mod factory; +/// Metadata garbage collection. +pub mod gc; /// Metadata state management shared by multi-backend runtime paths. pub mod meta; +/// V2 protocol initialization and full-data import. +pub mod migration; +/// V2 multi-write persistence and in-memory models. +pub mod model; +/// V2 multi-write persistence provider interfaces and implementations. +pub mod provider; +/// Logical path mapping and account partition routing. +pub mod router; +/// Foreground event queue and metadata flush worker. +pub mod runtime; /// Shared build-time types for multi-backend assembly. pub mod types; -pub use meta::{FsContextResolver, MetaStateStore}; +pub use meta::FsContextResolver; diff --git a/crates/ragfs/src/multibackend/model.rs b/crates/ragfs/src/multibackend/model.rs new file mode 100644 index 0000000000..85b782d0f6 --- /dev/null +++ b/crates/ragfs/src/multibackend/model.rs @@ -0,0 +1,647 @@ +//! V2 multi-write persistence and in-memory data models. +#![allow(missing_docs)] +use crate::core::errors::{Error, Result}; +use crate::multibackend::constants::{ + CHECKPOINT_DIR_PREFIX, HEAD_SEGMENT_FILE_PREFIX, MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS, + SEALED_SEGMENT_FILE_PREFIX, SEGMENT_FILE_EXTENSION, +}; +use crate::multibackend::router::validate_routes; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashSet}; +use uuid::Uuid; +macro_rules! ensure { ($condition:expr, $($arg:tt)*) => { if !$condition { return Err(Error::Serialization(format!($($arg)*))); } }; } +pub use crate::multibackend::constants::VBUCKETS; +const FORMAT_VERSION: u32 = 1; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ProtocolState { + pub protocol_version: ProtocolVersion, + pub status: ProtocolStatus, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum ProtocolVersion { + V2, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ProtocolStatus { + Migrating, + Stable, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PartitionsManifest { + pub version: u32, + pub epoch: u64, + pub vbuckets: u32, + pub partitions: BTreeMap, + pub routes: Vec, + pub directory_events: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PartitionEntry { + pub state: PartitionState, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum PartitionState { + Stable, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RouteEntry { + pub start: u32, + pub end: u32, + pub partition: u32, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PartitionManifest { + pub version: u32, + pub partition_id: u32, + pub epoch: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DirectoryEvent { + pub op_id: Uuid, + pub operation: DirectoryOperation, + pub source_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub destination_path: Option, + pub positions: Vec, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum DirectoryOperation { + RemoveTree, + MoveTree, + DeleteScope, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MarkerPosition { + pub partition_id: u32, + pub epoch: u64, + pub seq: u64, +} +#[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScopeKey { + pub account_id: String, + pub partition_id: u32, + pub epoch: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScopedSeq { + pub scope: ScopeKey, + pub seq: u64, +} +#[derive(Debug, Clone)] +pub struct PartitionContext { + pub scope: ScopeKey, + pub segment_dir: String, +} +#[derive(Debug, Clone)] +pub struct PendingEvent { + pub operation_id: Uuid, + pub account_id: String, + pub kind: PendingEventKind, + pub path: String, + pub destination_path: Option, + pub route_hint: Option, +} +#[derive(Debug, Clone)] +pub enum PendingEventKind { + Data(SegmentEventType), + DeleteAccount, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[repr(u8)] +pub enum SegmentEventType { + Write = 1, + Remove = 2, + RemoveTree = 3, + MoveTree = 4, + DeleteScope = 5, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SegmentRecord { + pub seq: u64, + pub event_type: SegmentEventType, + pub path: String, + pub op_id: Option, + pub destination_path: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SegmentManifest { + pub version: u32, + pub next_seq: u64, + pub segments: Vec, + pub backend_states: BTreeMap, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SegmentDescriptor { + pub path: String, + pub segment_from_seq: u64, + pub segment_to_seq: Option, + pub record_count: u32, + pub byte_len: u64, + pub checksum: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct BackendState { + pub synced_seq: u64, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FlushResult { + pub scope: ScopeKey, + pub first_seq: u64, + pub last_seq: u64, + pub sealed_segments: Vec, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedSegment { + pub records: Vec, + pub valid_bytes: u64, + pub truncated_tail: bool, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileState { + pub path: String, + pub latest_seq: ScopedSeq, + pub deleted: Option, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CheckpointNode { + pub path_fragment: String, + pub latest_seq: Option, + pub deleted: Option, + pub children: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CheckpointsManifest { + pub version: u32, + #[serde(deserialize_with = "Option::deserialize")] + pub latest_checkpoint: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct LatestCheckpoint { + pub path: String, + pub checkpoint_to_seq: u64, + pub updated_at_ns: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CheckpointManifest { + pub version: u32, + pub partition_id: u32, + pub epoch: u64, + pub checkpoint_from_seq: u64, + pub checkpoint_to_seq: u64, + pub created_at_ns: u64, + pub root: ChunkDescriptor, + pub checksum: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChunkDescriptor { + pub file: String, + pub root_path: String, + pub start_path: Option, + pub end_path: Option, + pub file_state_count: u32, + pub checksum: String, + pub chunks: Vec, +} +impl ProtocolState { + /// Validate the closed protocol version value. + pub fn validate(&self) -> Result<()> { + Ok(()) + } +} +impl PartitionsManifest { + /// Validate routing and directory marker invariants. + pub fn validate(&self) -> Result<()> { + validate_version("partitions manifest", self.version)?; + ensure!(self.epoch > 0, "partitions epoch must be positive"); + ensure!(self.vbuckets == VBUCKETS, "invalid vbucket count"); + ensure!(!self.partitions.is_empty(), "partitions must not be empty"); + for entry in self.partitions.values() { + entry.validate()?; + } + validate_routes(&self.routes, self.partitions.keys().copied())?; + let mut operation_ids = HashSet::new(); + for event in &self.directory_events { + ensure!( + operation_ids.insert(event.op_id), + "duplicate directory op_id" + ); + event.validate()?; + for position in &event.positions { + ensure!( + position.epoch == self.epoch, + "marker epoch does not match manifest" + ); + ensure!( + self.partitions + .get(&position.partition_id) + .is_some_and(|entry| entry.state == PartitionState::Stable), + "marker partition is unavailable" + ); + } + } + ensure!( + self.partitions + .values() + .all(|entry| entry.state == PartitionState::Stable), + "partition manifest contains an unsupported state" + ); + Ok(()) + } +} +impl PartitionEntry { + /// Validate the closed partition state value. + pub fn validate(&self) -> Result<()> { + Ok(()) + } +} +impl RouteEntry { + /// Validate one inclusive vbucket route range. + pub fn validate(&self) -> Result<()> { + ensure!( + self.start <= self.end && self.end < VBUCKETS, + "invalid route range" + ); + Ok(()) + } +} +impl PartitionManifest { + /// Validate one partition manifest version and epoch. + pub fn validate(&self) -> Result<()> { + validate_version("partition manifest", self.version)?; + ensure!(self.epoch > 0, "partition epoch must be positive"); + Ok(()) + } +} +impl DirectoryEvent { + /// Validate one directory operation and its marker positions. + pub fn validate(&self) -> Result<()> { + validate_path(&self.source_path)?; + match self.operation { + DirectoryOperation::RemoveTree | DirectoryOperation::DeleteScope => ensure!( + self.destination_path.is_none(), + "remove tree has a destination" + ), + DirectoryOperation::MoveTree => { + validate_path(self.destination_path.as_deref().ok_or_else(|| { + Error::Serialization("move tree requires a destination".into()) + })?)? + } + } + let mut partitions = HashSet::new(); + for position in &self.positions { + position.validate()?; + ensure!( + partitions.insert(position.partition_id), + "duplicate marker partition" + ); + } + Ok(()) + } +} +impl MarkerPosition { + /// Validate one persisted directory marker position. + pub fn validate(&self) -> Result<()> { + ensure!(self.epoch > 0 && self.seq > 0, "invalid marker position"); + Ok(()) + } +} +impl ScopeKey { + /// Validate an account-scoped partition identity. + pub fn validate(&self) -> Result<()> { + ensure!( + !self.account_id.trim().is_empty() && self.epoch > 0, + "invalid scope" + ); + Ok(()) + } + /// Validate this scope against its owning partitions manifest. + pub fn validate_against_manifest(&self, manifest: &PartitionsManifest) -> Result<()> { + self.validate()?; + ensure!( + manifest.partitions.contains_key(&self.partition_id), + "scope partition is unknown" + ); + ensure!( + self.epoch == manifest.epoch, + "scope epoch does not match manifest" + ); + Ok(()) + } +} +impl ScopedSeq { + /// Validate a scoped sequence value. + pub fn validate(&self) -> Result<()> { + self.scope.validate()?; + ensure!(self.seq > 0, "scoped sequence must be positive"); + Ok(()) + } +} +impl SegmentRecord { + /// Validate a persisted segment record. + pub fn validate(&self) -> Result<()> { + ensure!(self.seq > 0, "record sequence must be positive"); + validate_path(&self.path)?; + match self.event_type { + SegmentEventType::MoveTree => { + ensure!(self.op_id.is_some(), "move marker requires an op_id"); + validate_path(self.destination_path.as_deref().ok_or_else(|| { + Error::Serialization("move marker requires a destination".into()) + })?)?; + } + SegmentEventType::RemoveTree | SegmentEventType::DeleteScope => { + ensure!(self.op_id.is_some(), "remove marker requires an op_id"); + ensure!( + self.destination_path.is_none(), + "remove marker has a destination" + ); + } + _ => ensure!( + self.destination_path.is_none(), + "data event has a destination" + ), + } + Ok(()) + } +} +impl SegmentManifest { + /// Validate segment continuity and backend progress. + pub fn validate(&self) -> Result<()> { + validate_version("segment manifest", self.version)?; + ensure!(self.next_seq > 0, "next sequence must be positive"); + let (mut next, mut paths) = (None, HashSet::new()); + for (index, segment) in self.segments.iter().enumerate() { + segment.validate()?; + ensure!(paths.insert(&segment.path), "duplicate segment path"); + if let Some(expected) = next { + ensure!(segment.segment_from_seq == expected, "segment sequence gap"); + } + ensure!( + segment.segment_to_seq.is_some() || index + 1 == self.segments.len(), + "head segment must be last" + ); + next = segment + .segment_from_seq + .checked_add(u64::from(segment.record_count)); + ensure!(next.is_some(), "segment sequence overflow"); + } + if let Some(expected) = next { + ensure!( + self.next_seq == expected, + "next sequence does not follow segments" + ); + } + for (backend, state) in &self.backend_states { + ensure!(!backend.trim().is_empty(), "backend id must not be empty"); + state.validate(self.next_seq - 1)?; + } + Ok(()) + } +} +impl SegmentDescriptor { + /// Validate one segment summary. + pub fn validate(&self) -> Result<()> { + ensure!( + !self.path.is_empty() && self.segment_from_seq > 0, + "invalid segment identity" + ); + ensure!(self.byte_len >= 6, "segment byte length is too small"); + match self.segment_to_seq { + Some(to) => { + ensure!(to >= self.segment_from_seq, "segment range is invalid"); + ensure!( + self.path + == format!( + "{SEALED_SEGMENT_FILE_PREFIX}{:020}-{to:020}{SEGMENT_FILE_EXTENSION}", + self.segment_from_seq + ), + "sealed segment path does not match range" + ); + ensure!( + u64::from(self.record_count) == to - self.segment_from_seq + 1, + "sealed segment count does not match range" + ); + ensure!( + self.record_count <= MAX_SEGMENT_RECORDS as u32, + "sealed segment has too many records" + ); + validate_checksum(self.checksum.as_deref().ok_or_else(|| { + Error::Serialization("sealed segment requires a checksum".into()) + })?)?; + } + None => { + ensure!( + self.path + == format!( + "{HEAD_SEGMENT_FILE_PREFIX}{:020}{SEGMENT_FILE_EXTENSION}", + self.segment_from_seq + ), + "head segment path does not match start sequence" + ); + ensure!(self.checksum.is_none(), "head segment has a checksum"); + } + } + ensure!( + self.segment_from_seq + .checked_add(u64::from(self.record_count)) + .is_some(), + "segment sequence overflow" + ); + Ok(()) + } +} +impl BackendState { + /// Validate backend progress against the committed sequence. + pub fn validate(&self, committed_seq: u64) -> Result<()> { + ensure!( + self.synced_seq <= committed_seq, + "backend progress is ahead" + ); + Ok(()) + } +} +impl FileState { + /// Validate a checkpoint file state against its owning scope. + pub fn validate(&self, expected_scope: &ScopeKey) -> Result<()> { + validate_path(&self.path)?; + self.latest_seq.validate()?; + ensure!( + &self.latest_seq.scope == expected_scope, + "file state crosses checkpoint scope" + ); + validate_tombstone(self.deleted) + } +} +impl CheckpointNode { + /// Validate one checkpoint radix tree and its tombstones. + pub fn validate(&self) -> Result<()> { + ensure!( + self.latest_seq.is_none_or(|seq| seq > 0), + "invalid checkpoint node sequence" + ); + validate_tombstone(self.deleted)?; + ensure!( + self.deleted.is_none() || self.latest_seq.is_some(), + "tombstone requires a sequence" + ); + let mut previous = None; + for child in &self.children { + child.validate()?; + ensure!( + previous.is_none_or(|fragment| fragment < child.path_fragment.as_str()), + "checkpoint children are not strictly ordered" + ); + previous = Some(child.path_fragment.as_str()); + } + Ok(()) + } +} +impl CheckpointsManifest { + /// Validate the checkpoint publication pointer. + pub fn validate(&self) -> Result<()> { + validate_version("checkpoints manifest", self.version)?; + if let Some(latest) = &self.latest_checkpoint { + latest.validate()?; + } + Ok(()) + } +} +impl LatestCheckpoint { + /// Validate one published checkpoint reference. + pub fn validate(&self) -> Result<()> { + let name = self.path.strip_suffix('/').ok_or_else(|| { + Error::Serialization("checkpoint pointer must reference a directory".into()) + })?; + ensure!( + is_checkpoint_directory_name(name), + "invalid checkpoint reference" + ); + Ok(()) + } +} + +/// Return whether a single directory name identifies a published checkpoint. +pub(crate) fn is_checkpoint_directory_name(name: &str) -> bool { + !name.contains('/') && name.starts_with(CHECKPOINT_DIR_PREFIX) +} + +impl CheckpointManifest { + /// Validate a checkpoint manifest and its chunk tree. + pub fn validate(&self) -> Result<()> { + validate_version("checkpoint manifest", self.version)?; + ensure!(self.epoch > 0, "checkpoint epoch must be positive"); + ensure!( + self.checkpoint_from_seq == 1 && self.checkpoint_to_seq == 0 + || self.checkpoint_from_seq > 0 + && self.checkpoint_from_seq <= self.checkpoint_to_seq, + "invalid checkpoint sequence range" + ); + validate_checksum(&self.checksum)?; + self.root.validate() + } + /// Validate this checkpoint against its owning scope. + pub fn validate_for_scope(&self, scope: &ScopeKey) -> Result<()> { + self.validate()?; + scope.validate()?; + ensure!( + self.partition_id == scope.partition_id, + "checkpoint partition does not match scope" + ); + ensure!( + self.epoch == scope.epoch, + "checkpoint epoch does not match scope" + ); + Ok(()) + } +} +impl ChunkDescriptor { + /// Validate a checkpoint chunk tree, including unique files and ranges. + pub fn validate(&self) -> Result<()> { + self.validate_inner(true, &mut HashSet::new()) + } + /// Validate one chunk and recurse into its direct children. + fn validate_inner(&self, root: bool, files: &mut HashSet) -> Result<()> { + ensure!( + !self.file.is_empty() && files.insert(self.file.clone()), + "invalid chunk file" + ); + validate_path(&self.root_path)?; + ensure!( + self.file_state_count <= MAX_CHECKPOINT_NODES as u32, + "chunk has too many file states" + ); + validate_checksum(&self.checksum)?; + let (range_start, range_end) = + match (root, self.start_path.as_deref(), self.end_path.as_deref()) { + (true, None, None) => (self.root_path.as_str(), None), + (true, _, _) => { + return Err(Error::Serialization( + "root chunk must not have bounds".into(), + )); + } + (false, Some(start), end) => { + if let Some(end) = end { + ensure!(start < end, "chunk range is reversed"); + } + (start, end) + } + (false, None, _) => { + return Err(Error::Serialization( + "non-root chunk requires start bound".into(), + )); + } + }; + let mut expected_start = Some(range_start); + for child in &self.chunks { + child.validate_inner(false, files)?; + let child_start = child.start_path.as_deref().unwrap(); + ensure!( + expected_start.is_some_and(|start| start == child_start), + "chunk ranges are not contiguous" + ); + expected_start = child.end_path.as_deref(); + } + if !self.chunks.is_empty() { + ensure!( + expected_start == range_end, + "chunk ranges are not contiguous" + ); + } + Ok(()) + } +} +/// Validate a supported persisted format version. +fn validate_version(name: &str, version: u32) -> Result<()> { + ensure!( + version == FORMAT_VERSION, + "unsupported {name} version {version}" + ); + Ok(()) +} +/// Validate a complete logical path. +fn validate_path(path: &str) -> Result<()> { + ensure!(path.starts_with('/'), "logical path must be absolute"); + Ok(()) +} +/// Validate a lowercase SHA-256 hexadecimal checksum. +fn validate_checksum(checksum: &str) -> Result<()> { + ensure!( + checksum.len() == 64 + && checksum + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "invalid SHA-256 checksum" + ); + Ok(()) +} +/// Validate an optional deletion tombstone. +fn validate_tombstone(deleted: Option) -> Result<()> { + ensure!(deleted != Some(false), "tombstone may only be true"); + Ok(()) +} + +#[cfg(test)] +#[path = "tests/test_model.rs"] +mod tests; diff --git a/crates/ragfs/src/multibackend/provider/cache.rs b/crates/ragfs/src/multibackend/provider/cache.rs new file mode 100644 index 0000000000..88161ba58d --- /dev/null +++ b/crates/ragfs/src/multibackend/provider/cache.rs @@ -0,0 +1,1555 @@ +//! CacheRuntime-backed V2 multi-write persistence. + +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use serde::Deserialize; +use uuid::Uuid; + +use crate::cache_runtime::{ + CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue, +}; +use crate::core::errors::{Error, Result}; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex}; +use crate::multibackend::constants::{ + HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX, + SEGMENT_FILE_EXTENSION, +}; +use crate::multibackend::meta::MetadataStore; +use crate::multibackend::model::{ + BackendState, CheckpointsManifest, FlushResult, MarkerPosition, PartitionContext, + PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentDescriptor, + SegmentEventType, SegmentManifest, SegmentRecord, +}; +use crate::multibackend::router::AccountRouter; + +use super::{read_sealed_segment, select_record_range, MultiWriteProvider}; + +const FLUSH_SCRIPT_ID: &str = "multiwrite.flush.v1"; +const SEAL_SCRIPT_ID: &str = "multiwrite.seal.v1"; +const ADVANCE_SCRIPT_ID: &str = "multiwrite.advance.v1"; + +const FLUSH_SCRIPT: &str = r#" +if redis.call("EXISTS", KEYS[1]) == 0 then + return { "not_found" } +end +if redis.call("HGET", KEYS[1], "account") ~= ARGV[1] + or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2] + or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then + return { "scope_mismatch" } +end +local requested = #ARGV - 6 +if ARGV[4] == "1" and requested ~= 1 then + return { "invalid", "marker flush requires exactly one record" } +end +local owner = redis.call("GET", KEYS[3]) +if ARGV[6] == "1" then + if owner ~= ARGV[5] then + return { "fenced" } + end +elseif owner and owner ~= ARGV[5] then + return { "busy" } +else + redis.call("SET", KEYS[3], ARGV[5], "PX", 30000) +end +redis.call("PEXPIRE", KEYS[3], 30000) +local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq")) +if marker_seq ~= 0 then + return { "seal_required" } +end +local head_len = redis.call("LLEN", KEYS[2]) +local capacity = 8192 - head_len +if capacity <= 0 then + return { "seal_required" } +end +local count = math.min(requested, capacity) +local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq")) +local first_seq = next_seq +for index = 1, count do + local record = cjson.decode(ARGV[index + 6]) + record.seq = next_seq + redis.call("RPUSH", KEYS[2], cjson.encode(record)) + next_seq = next_seq + 1 +end +if ARGV[4] == "1" then + marker_seq = next_seq - 1 +end +local next_seq_text = string.format("%.0f", next_seq) +local marker_seq_text = string.format("%.0f", marker_seq) +redis.call("HSET", KEYS[1], "next_seq", next_seq_text, + "marker_seq", marker_seq_text) +local needs_seal = 0 +if redis.call("LLEN", KEYS[2]) == 8192 or marker_seq ~= 0 then + needs_seal = 1 +end +if needs_seal == 0 then + redis.call("DEL", KEYS[3]) +else + redis.call("PEXPIRE", KEYS[3], 30000) +end +return { "ok", string.format("%.0f", first_seq), + string.format("%.0f", next_seq - 1), count, needs_seal } +"#; + +const SEAL_SCRIPT: &str = r#" +local mode = ARGV[1] +if mode == "bootstrap" then + if redis.call("EXISTS", KEYS[1]) ~= 0 then + return { "exists" } + end + redis.call("DEL", KEYS[1]) + redis.call("HSET", KEYS[1], + "account", ARGV[2], + "partition", ARGV[3], + "epoch", ARGV[4], + "version", ARGV[5], + "next_seq", ARGV[6], + "marker_seq", "0", + "segments", ARGV[7], + "backend_states", ARGV[8]) + redis.call("DEL", KEYS[2]) + redis.call("DEL", KEYS[3]) + return { "ok" } +end +if redis.call("EXISTS", KEYS[1]) == 0 then + return { "not_found" } +end +if redis.call("HGET", KEYS[1], "account") ~= ARGV[2] + or redis.call("HGET", KEYS[1], "partition") ~= ARGV[3] + or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[4] then + return { "scope_mismatch" } +end +if mode == "read" then + return { + "ok", + redis.call("HGET", KEYS[1], "account"), + redis.call("HGET", KEYS[1], "partition"), + redis.call("HGET", KEYS[1], "epoch"), + redis.call("HGET", KEYS[1], "version"), + redis.call("HGET", KEYS[1], "next_seq"), + redis.call("HGET", KEYS[1], "marker_seq"), + redis.call("HGET", KEYS[1], "segments"), + redis.call("HGET", KEYS[1], "backend_states") + } +end +local segments = cjson.decode(redis.call("HGET", KEYS[1], "segments")) +if mode == "commit" then + local first_seq = tonumber(ARGV[5]) + local last_seq = tonumber(ARGV[6]) + local count = tonumber(ARGV[7]) + local descriptor = cjson.decode(ARGV[8]) + local owner = ARGV[9] + local keep_owner = ARGV[10] + for _, current in ipairs(segments) do + if current.path == descriptor.path then + if current.segment_from_seq == descriptor.segment_from_seq + and current.segment_to_seq == descriptor.segment_to_seq + and current.record_count == descriptor.record_count + and current.byte_len == descriptor.byte_len + and current.checksum == descriptor.checksum then + if owner ~= "" and redis.call("GET", KEYS[3]) == owner then + if keep_owner == "1" then + redis.call("PEXPIRE", KEYS[3], 30000) + else + redis.call("DEL", KEYS[3]) + end + end + return { "already" } + end + return { "invalid", "sealed descriptor path conflicts" } + end + end + local current_owner = redis.call("GET", KEYS[3]) + if owner == "" then + if current_owner then + return { "busy" } + end + elseif current_owner ~= owner then + return { "fenced" } + end + if #segments > 0 + and segments[#segments].segment_to_seq + 1 ~= first_seq then + return { "invalid", "sealed descriptor is not contiguous" } + end + if count ~= last_seq - first_seq + 1 then + return { "invalid", "sealed descriptor count is invalid" } + end + if redis.call("LLEN", KEYS[2]) < count then + return { "changed" } + end + for index = 1, count do + if redis.call("LINDEX", KEYS[2], index - 1) ~= ARGV[index + 10] then + return { "changed" } + end + end + segments[#segments + 1] = descriptor + redis.call("HSET", KEYS[1], "segments", cjson.encode(segments)) + redis.call("LTRIM", KEYS[2], count, -1) + local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq")) + if marker_seq ~= 0 and marker_seq <= last_seq then + redis.call("HSET", KEYS[1], "marker_seq", "0") + end + if owner ~= "" then + if keep_owner == "1" then + redis.call("PEXPIRE", KEYS[3], 30000) + else + redis.call("DEL", KEYS[3]) + end + end + return { "ok" } +end +if mode == "prune" then + local requested = cjson.decode(ARGV[5]) + local requested_set = {} + for _, path in ipairs(requested) do + if requested_set[path] then + return { "invalid", "prune paths contain duplicates" } + end + requested_set[path] = true + end + local removed = {} + local kept = {} + for _, descriptor in ipairs(segments) do + if requested_set[descriptor.path] then + removed[#removed + 1] = descriptor + requested_set[descriptor.path] = nil + else + kept[#kept + 1] = descriptor + end + end + for path, _ in pairs(requested_set) do + return { "invalid", "sealed segment is not referenced: " .. path } + end + for index = 2, #kept do + if kept[index].segment_from_seq ~= kept[index - 1].segment_to_seq + 1 then + return { "invalid", "prune would create a segment gap" } + end + end + local head = redis.call("LINDEX", KEYS[2], 0) + if head and #kept > 0 then + local first_head = cjson.decode(head) + if first_head.seq ~= kept[#kept].segment_to_seq + 1 then + return { "invalid", "prune would create a head gap" } + end + elseif not head and #kept > 0 then + local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq")) + if next_seq ~= kept[#kept].segment_to_seq + 1 then + return { "invalid", "prune would detach next_seq" } + end + end + local encoded_kept = "[]" + if #kept > 0 then + encoded_kept = cjson.encode(kept) + end + local encoded_removed = "[]" + if #removed > 0 then + encoded_removed = cjson.encode(removed) + end + redis.call("HSET", KEYS[1], "segments", encoded_kept) + return { "ok", encoded_removed } +end +return { "invalid", "unknown seal operation" } +"#; + +const ADVANCE_SCRIPT: &str = r#" +if redis.call("EXISTS", KEYS[1]) == 0 then + return { "not_found" } +end +if redis.call("HGET", KEYS[1], "account") ~= ARGV[1] + or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2] + or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then + return { "scope_mismatch" } +end +local states = cjson.decode(redis.call("HGET", KEYS[1], "backend_states")) +local state = states[ARGV[4]] +if not state then + return { "invalid", "unknown backend id: " .. ARGV[4] } +end +local synced_seq = tonumber(ARGV[5]) +if synced_seq ~= 0 and synced_seq < state.synced_seq then + return { "invalid", "backend progress cannot regress" } +end +local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq")) +if synced_seq >= next_seq then + return { "invalid", "backend progress must remain below next_seq" } +end +state.synced_seq = synced_seq +redis.call("HSET", KEYS[1], "backend_states", cjson.encode(states)) +return { "ok" } +"#; + +const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[ + ScriptDefinition { + id: FLUSH_SCRIPT_ID, + redis_lua: FLUSH_SCRIPT, + }, + ScriptDefinition { + id: SEAL_SCRIPT_ID, + redis_lua: SEAL_SCRIPT, + }, + ScriptDefinition { + id: ADVANCE_SCRIPT_ID, + redis_lua: ADVANCE_SCRIPT, + }, +]; + +/// Cache keys owned by one account partition. +pub(crate) struct CachePartitionKeys { + pub(crate) state: String, + pub(crate) head: String, + pub(crate) flush_gate: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CacheState { + scope: ScopeKey, + version: u32, + next_seq: u64, + marker_seq: u64, + segments: Vec, + backend_states: BTreeMap, +} + +struct FlushScriptResult { + first_seq: u64, + last_seq: u64, + appended: usize, + needs_seal: bool, +} + +enum FlushScriptOutcome { + Appended(FlushScriptResult), + SealRequired, + Busy, +} + +/// Persists V2 mutable state through CacheRuntime and immutable segments on the primary filesystem. +pub struct CacheProvider { + store: Arc, + router: AccountRouter, + runtime: Arc, + namespace: String, +} + +impl CacheProvider { + /// Creates a cache-backed metadata provider. + pub fn new( + store: Arc, + runtime: Arc, + namespace: String, + ) -> Result { + Self::build(store, runtime, namespace) + } + + /// Validates runtime capabilities and registers stable provider scripts. + fn build( + store: Arc, + runtime: Arc, + namespace: String, + ) -> Result { + Self::validate_namespace(&namespace)?; + runtime + .require_operations(&[CacheOperation::Lrange, CacheOperation::ExecuteScript]) + .map_err(|error| { + Error::config(format!( + "Cache-backed multi-write provider is incompatible: {error}" + )) + })?; + for definition in SCRIPT_DEFINITIONS { + runtime.register_script(*definition).map_err(|error| { + Error::config(format!( + "Cache-backed multi-write script registration failed: {error}" + )) + })?; + } + Ok(Self { + router: AccountRouter::new(store.clone()), + store, + runtime, + namespace, + }) + } + + /// Validate the Redis hash tag namespace used by multi-write cache keys. + fn validate_namespace(namespace: &str) -> Result<()> { + if namespace.trim().is_empty() || namespace.contains(['{', '}']) { + return Err(Error::config( + "cache-backed multi-write namespace must be non-empty and must not contain '{' or '}'", + )); + } + Ok(()) + } + + /// Builds deterministic same-slot keys for one validated scope. + pub(crate) fn keys_for_scope(&self, scope: &ScopeKey) -> Result { + scope.validate()?; + if matches!(scope.account_id.as_str(), "." | "..") + || scope.account_id.contains('/') + || scope + .account_id + .bytes() + .any(|byte| byte.is_ascii_control() || matches!(byte, b'{' | b'}')) + { + return Err(Error::invalid_path( + "Cache-backed multi-write account id contains an invalid key character", + )); + } + let prefix = format!( + "{{{}}}:ov:multiwrite:{}:{}", + self.namespace, scope.account_id, scope.partition_id + ); + Ok(CachePartitionKeys { + state: format!("{prefix}:state"), + head: format!("{prefix}:head"), + flush_gate: format!("{prefix}:flush-gate"), + }) + } + + /// Validates normal flush input before routing or cache access. + fn validate_flush_events(events: &[PendingEvent]) -> Result<()> { + let first = events + .first() + .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?; + for event in events { + if event.account_id != first.account_id { + return Err(Error::invalid_operation( + "flush events must belong to one account", + )); + } + if is_multiwrite_internal_path(&event.path) { + return Err(Error::invalid_path(format!( + "cannot flush internal path: {}", + event.path + ))); + } + match event.kind { + PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {} + _ => { + return Err(Error::invalid_operation( + "normal flush accepts only write and remove events", + )) + } + } + if event.destination_path.is_some() { + return Err(Error::invalid_operation( + "normal data event cannot have a destination", + )); + } + } + Ok(()) + } + + /// Routes every event and requires one current partition scope. + async fn route_events(&self, events: &[PendingEvent]) -> Result { + let mut selected = None; + for event in events { + let current = self.router.route(&event.account_id, &event.path).await?; + if selected + .as_ref() + .is_some_and(|existing: &PartitionContext| existing.scope != current.scope) + { + return Err(Error::invalid_operation( + "flush events route to different partition scopes", + )); + } + selected.get_or_insert(current); + } + selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event")) + } + + /// Validates a scope against its current account partitions manifest. + async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> { + self.keys_for_scope(scope)?; + let path = self.store.paths().account_manifest(&scope.account_id)?.0; + let manifest: PartitionsManifest = self.store.read_json(&path).await?; + manifest.validate()?; + scope.validate_against_manifest(&manifest) + } + + /// Converts one pending event into a validated record template. + fn record(event: &PendingEvent) -> Result { + let event_type = match event.kind { + PendingEventKind::Data(kind) => kind, + PendingEventKind::DeleteAccount => { + return Err(Error::invalid_operation( + "account deletion is not a segment record", + )) + } + }; + let marker = matches!( + event_type, + SegmentEventType::RemoveTree + | SegmentEventType::MoveTree + | SegmentEventType::DeleteScope + ); + let record = SegmentRecord { + seq: 1, + event_type, + path: event.path.clone(), + op_id: marker.then_some(event.operation_id), + destination_path: event.destination_path.clone(), + }; + record.validate()?; + encode_segment(std::slice::from_ref(&record))?; + Ok(record) + } + + /// Validates one marker against its caller-selected scope. + fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> { + if event.account_id != scope.account_id { + return Err(Error::invalid_operation( + "marker account does not match supplied scope", + )); + } + match event.kind { + PendingEventKind::Data( + SegmentEventType::RemoveTree + | SegmentEventType::MoveTree + | SegmentEventType::DeleteScope, + ) => {} + _ => { + return Err(Error::invalid_operation( + "append_marker accepts only directory markers", + )) + } + } + if is_multiwrite_internal_path(&event.path) { + return Err(Error::invalid_path(format!( + "cannot append marker for internal path: {}", + event.path + ))); + } + if !matches!( + event.kind, + PendingEventKind::Data(SegmentEventType::DeleteScope) + ) { + self.store + .paths() + .backend_path(&scope.account_id, &event.path)?; + } + if let Some(destination) = &event.destination_path { + if is_multiwrite_internal_path(destination) { + return Err(Error::invalid_path(format!( + "cannot append marker for internal destination: {destination}" + ))); + } + self.store + .paths() + .backend_path(&scope.account_id, destination)?; + } + Self::record(event).map(|_| ()) + } + + /// Executes one registered script and requires an array result. + async fn execute_array( + &self, + script_id: &str, + keys: &CachePartitionKeys, + args: Vec, + ) -> Result> { + let result = self + .runtime + .execute_script(ScriptRequest { + script_id: script_id.to_string(), + keys: vec![ + keys.state.clone(), + keys.head.clone(), + keys.flush_gate.clone(), + ], + args, + }) + .await + .and_then(|result| result.decode()) + .map_err(|error| Self::cache_error(script_id, error))?; + match result { + ScriptValue::Array(values) => Ok(values), + other => Err(Error::Serialization(format!( + "Cache-backed multi-write {script_id} returned non-array result: {other:?}" + ))), + } + } + + /// Executes one bounded atomic append attempt. + async fn flush_once( + &self, + scope: &ScopeKey, + records: &[SegmentRecord], + marker: bool, + owner: &str, + continuing: bool, + ) -> Result { + let keys = self.keys_for_scope(scope)?; + let mut args = Self::scope_args(scope); + args.push(Bytes::from(if marker { "1" } else { "0" })); + args.push(Bytes::from(owner.to_string())); + args.push(Bytes::from(if continuing { "1" } else { "0" })); + for record in records { + args.push(Bytes::from(serde_json::to_vec(record)?)); + } + let values = self.execute_array(FLUSH_SCRIPT_ID, &keys, args).await?; + match Self::status(&values, "flush")? { + "ok" => Ok(FlushScriptOutcome::Appended(FlushScriptResult { + first_seq: Self::u64_value(&values, 1, "flush first_seq")?, + last_seq: Self::u64_value(&values, 2, "flush last_seq")?, + appended: Self::usize_value(&values, 3, "flush appended count")?, + needs_seal: Self::u64_value(&values, 4, "flush seal flag")? == 1, + })), + "seal_required" => Ok(FlushScriptOutcome::SealRequired), + "busy" => Ok(FlushScriptOutcome::Busy), + "fenced" => Err(Error::would_block( + "cache-backed flush owner was fenced by another writer", + )), + "not_found" => Err(Error::not_found(keys.state)), + "scope_mismatch" => Err(Error::invalid_operation( + "Cache-backed multi-write state scope does not match", + )), + "invalid" => Err(Self::script_invalid(&values, "flush")), + other => Err(Self::unknown_status("flush", other)), + } + } + + /// Revalidates routing and executes one append while holding the account manifest lock. + async fn flush_routed_once( + &self, + events: &[PendingEvent], + expected_scope: Option<&ScopeKey>, + records: &[SegmentRecord], + owner: &str, + continuing: bool, + ) -> Result<(PartitionContext, FlushScriptOutcome)> { + let account_id = &events + .first() + .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))? + .account_id; + let manifest_path = self.store.paths().account_manifest(account_id)?.0; + let lock_timeout = self + .store + .pathlock_manager() + .default_lock_timeout() + .max(std::time::Duration::from_secs(30)); + let lease = self + .store + .pathlock_manager() + .acquire_exact(&manifest_path, lock_timeout, None) + .await?; + let operation = async { + let route = self.route_events(events).await?; + if expected_scope.is_some_and(|scope| scope != &route.scope) { + return Err(Error::would_block( + "multi-write route changed during cache-backed flush", + )); + } + let outcome = self + .flush_once(&route.scope, records, false, owner, continuing) + .await?; + Ok((route, outcome)) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + match (operation, release) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + (Ok(result), Ok(())) => Ok(result), + } + } + + /// Revalidates a supplied scope and appends one marker under its manifest lock. + async fn flush_marker_once( + &self, + scope: &ScopeKey, + record: &SegmentRecord, + owner: &str, + continuing: bool, + ) -> Result { + let manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0; + let lock_timeout = self + .store + .pathlock_manager() + .default_lock_timeout() + .max(std::time::Duration::from_secs(30)); + let lease = self + .store + .pathlock_manager() + .acquire_exact(&manifest_path, lock_timeout, None) + .await?; + let operation = async { + self.validate_scope(scope).await?; + self.flush_once(scope, std::slice::from_ref(record), true, owner, continuing) + .await + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + match (operation, release) { + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + (Ok(result), Ok(())) => Ok(result), + } + } + + /// Reads one stable cache state and head-list snapshot. + async fn snapshot( + &self, + scope: &ScopeKey, + ) -> Result<(CacheState, Vec, Vec)> { + self.validate_scope(scope).await?; + let keys = self.keys_for_scope(scope)?; + loop { + let before = self.read_state_once(scope, &keys).await?; + let raw_records = self + .runtime + .lrange(&keys.head, 0, -1) + .await + .map_err(|error| Self::cache_error("read head", error))?; + let after = self.read_state_once(scope, &keys).await?; + if before == after { + let records = raw_records + .iter() + .map(|raw| { + serde_json::from_slice::(raw) + .map_err(|error| Error::Serialization(error.to_string())) + }) + .collect::>>()?; + Self::validate_snapshot(&before, &records)?; + return Ok((before, records, raw_records)); + } + } + } + + /// Reads one cache state value through the shared seal script. + async fn read_state_once( + &self, + scope: &ScopeKey, + keys: &CachePartitionKeys, + ) -> Result { + let mut args = vec![Bytes::from_static(b"read")]; + args.extend(Self::scope_args(scope)); + let values = self.execute_array(SEAL_SCRIPT_ID, keys, args).await?; + match Self::status(&values, "read state")? { + "ok" => { + let state = CacheState { + scope: ScopeKey { + account_id: Self::string_value(&values, 1, "state account")?.to_string(), + partition_id: Self::u32_value(&values, 2, "state partition")?, + epoch: Self::u64_value(&values, 3, "state epoch")?, + }, + version: Self::u32_value(&values, 4, "state version")?, + next_seq: Self::u64_value(&values, 5, "state next_seq")?, + marker_seq: Self::u64_value(&values, 6, "state marker_seq")?, + segments: Self::json_value(&values, 7, "state segments")?, + backend_states: Self::json_value(&values, 8, "state backend states")?, + }; + Self::validate_state(scope, &state)?; + Ok(state) + } + "not_found" => Err(Error::not_found(keys.state.clone())), + "scope_mismatch" => Err(Error::invalid_operation( + "Cache-backed multi-write state scope does not match", + )), + other => Err(Self::unknown_status("read state", other)), + } + } + + /// Validates cache state fields that do not depend on the mutable head. + fn validate_state(scope: &ScopeKey, state: &CacheState) -> Result<()> { + if &state.scope != scope { + return Err(Error::Serialization( + "Cache-backed multi-write state has the wrong scope".to_string(), + )); + } + if state.version != 1 || state.next_seq == 0 { + return Err(Error::Serialization( + "Cache-backed multi-write state has invalid version or next_seq".to_string(), + )); + } + let mut previous_to = None; + let mut paths = HashSet::new(); + for descriptor in &state.segments { + descriptor.validate()?; + if descriptor.segment_to_seq.is_none() || !paths.insert(&descriptor.path) { + return Err(Error::Serialization( + "cache state contains an invalid sealed descriptor".to_string(), + )); + } + if previous_to.is_some_and(|to| descriptor.segment_from_seq != to + 1) { + return Err(Error::Serialization( + "cache sealed descriptors are not contiguous".to_string(), + )); + } + previous_to = descriptor.segment_to_seq; + } + for (backend, backend_state) in &state.backend_states { + if backend.trim().is_empty() { + return Err(Error::Serialization( + "cache backend id must not be empty".to_string(), + )); + } + backend_state.validate(state.next_seq - 1)?; + } + if state.marker_seq >= state.next_seq { + return Err(Error::Serialization( + "cache marker is ahead of next_seq".to_string(), + )); + } + Ok(()) + } + + /// Validates head ordering and its continuity with cache manifest state. + fn validate_snapshot(state: &CacheState, records: &[SegmentRecord]) -> Result<()> { + if records.len() > MAX_SEGMENT_RECORDS { + return Err(Error::Serialization( + "cache head contains too many records".to_string(), + )); + } + for (index, record) in records.iter().enumerate() { + record.validate()?; + if index > 0 && record.seq != records[index - 1].seq + 1 { + return Err(Error::Serialization( + "cache head contains a sequence gap".to_string(), + )); + } + } + if let Some(first) = records.first() { + if state + .segments + .last() + .and_then(|descriptor| descriptor.segment_to_seq) + .is_some_and(|last| first.seq != last + 1) + { + return Err(Error::Serialization( + "cache head does not follow sealed segments".to_string(), + )); + } + if records.last().unwrap().seq.checked_add(1) != Some(state.next_seq) { + return Err(Error::Serialization( + "cache head does not end at next_seq".to_string(), + )); + } + } else if state + .segments + .last() + .and_then(|descriptor| descriptor.segment_to_seq) + .is_some_and(|last| last.checked_add(1) != Some(state.next_seq)) + { + return Err(Error::Serialization( + "cache next_seq does not follow sealed segments".to_string(), + )); + } + if state.marker_seq != 0 + && records.last().map(|record| record.seq) != Some(state.marker_seq) + { + return Err(Error::Serialization( + "cache marker is not the final head record".to_string(), + )); + } + Ok(()) + } + + /// Converts a stable snapshot into the public validated manifest. + fn manifest(state: CacheState, records: &[SegmentRecord]) -> Result { + let mut manifest = SegmentManifest { + version: state.version, + next_seq: state.next_seq, + segments: state.segments, + backend_states: state.backend_states, + }; + if let Some(first) = records.first() { + let bytes = encode_segment(records)?; + manifest.segments.push(SegmentDescriptor { + path: Self::head_name(first.seq), + segment_from_seq: first.seq, + segment_to_seq: None, + record_count: records.len() as u32, + byte_len: bytes.len() as u64, + checksum: None, + }); + } + manifest.validate()?; + Ok(manifest) + } + + /// Persists one immutable segment snapshot on the shared raw primary. + async fn persist_sealed( + &self, + scope: &ScopeKey, + records: &[SegmentRecord], + ) -> Result { + let first = records + .first() + .ok_or_else(|| Error::invalid_operation("cannot seal an empty cache head"))? + .seq; + let last = records.last().unwrap().seq; + let bytes = encode_segment(records)?; + let descriptor = SegmentDescriptor { + path: Self::sealed_name(first, last), + segment_from_seq: first, + segment_to_seq: Some(last), + record_count: records.len() as u32, + byte_len: bytes.len() as u64, + checksum: Some(sha256_hex(&bytes)), + }; + let directory = self + .store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + self.store + .write_immutable(&format!("{directory}/{}", descriptor.path), &bytes) + .await?; + Ok(descriptor) + } + + /// Commits one immutable descriptor and trims its exact cache head prefix. + pub(crate) async fn commit_seal( + &self, + scope: &ScopeKey, + raw_records: &[Bytes], + descriptor: &SegmentDescriptor, + owner: Option<&str>, + keep_owner: bool, + ) -> Result<()> { + let keys = self.keys_for_scope(scope)?; + let mut args = vec![Bytes::from_static(b"commit")]; + args.extend(Self::scope_args(scope)); + args.push(Bytes::from(descriptor.segment_from_seq.to_string())); + args.push(Bytes::from(descriptor.segment_to_seq.unwrap().to_string())); + args.push(Bytes::from(descriptor.record_count.to_string())); + args.push(Bytes::from(serde_json::to_vec(descriptor)?)); + args.push(Bytes::from(owner.unwrap_or_default().to_string())); + args.push(Bytes::from(if keep_owner { "1" } else { "0" })); + args.extend(raw_records.iter().cloned()); + let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?; + match Self::status(&values, "seal")? { + "ok" | "already" => Ok(()), + "not_found" => Err(Error::not_found(keys.state)), + "busy" => Err(Error::would_block("cache head is owned by an active flush")), + "fenced" => Err(Error::would_block( + "cache seal owner was fenced by another writer", + )), + "changed" => Err(Error::would_block( + "cache head changed while committing a sealed prefix", + )), + "scope_mismatch" => Err(Error::invalid_operation( + "Cache-backed multi-write state scope does not match", + )), + "invalid" => Err(Self::script_invalid(&values, "seal")), + other => Err(Self::unknown_status("seal", other)), + } + } + + /// Seals one snapshot while enforcing an optional flush-owner fence. + async fn seal_head_with_owner( + &self, + scope: &ScopeKey, + owner: Option<&str>, + keep_owner: bool, + ) -> Result> { + let (_, records, raw_records) = self.snapshot(scope).await?; + if records.is_empty() { + return Ok(None); + } + let directory = self + .store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + let path = format!( + "{directory}/{}", + Self::sealed_name(records.first().unwrap().seq, records.last().unwrap().seq) + ); + let lease = self + .store + .pathlock_manager() + .acquire_exact( + &path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(std::time::Duration::from_secs(30)), + None, + ) + .await?; + let operation = async { + let descriptor = self.persist_sealed(scope, &records).await?; + self.commit_seal(scope, &raw_records, &descriptor, owner, keep_owner) + .await?; + Ok(Some(descriptor)) + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + match operation { + Ok(result) => release.map(|()| result), + Err(error) => Err(error), + } + } + + /// Returns whether one marker is the end of a referenced sealed segment. + async fn marker_is_sealed(&self, scope: &ScopeKey, seq: u64) -> Result { + Ok(self + .read_manifest(scope) + .await? + .segments + .iter() + .any(|descriptor| descriptor.segment_to_seq == Some(seq))) + } + + /// Returns the deterministic mutable head filename. + fn head_name(from_seq: u64) -> String { + format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}") + } + + /// Returns the deterministic immutable segment filename. + fn sealed_name(from_seq: u64, to_seq: u64) -> String { + format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}") + } + + /// Rebuild missing mutable state from the latest checkpoint and retained segments. + async fn recovery_manifest( + &self, + scope: &ScopeKey, + mut manifest: SegmentManifest, + ) -> Result { + manifest.validate()?; + for state in manifest.backend_states.values_mut() { + state.synced_seq = 0; + } + manifest.segments.clear(); + manifest.next_seq = match self + .store + .read_json::( + &self + .store + .paths() + .checkpoints_manifest(&scope.account_id, scope.partition_id)? + .0, + ) + .await + { + Ok(pointer) => { + pointer.validate()?; + pointer.latest_checkpoint.map_or(1, |latest| { + latest.checkpoint_to_seq.checked_add(1).unwrap_or(u64::MAX) + }) + } + Err(Error::NotFound(_)) => 1, + Err(error) => return Err(error), + }; + let directory = self + .store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + let mut candidates = match self.store.list_directory(&directory).await { + Ok(entries) => entries + .into_iter() + .filter_map(|entry| { + (!entry.is_dir) + .then(|| Self::sealed_range(&entry.name).map(|range| (range, entry.name))) + .flatten() + }) + .collect::>(), + Err(Error::NotFound(_)) => Vec::new(), + Err(error) => return Err(error), + }; + candidates.sort_by_key(|(range, _)| *range); + for ((from, to), path) in candidates { + if to < manifest.next_seq { + continue; + } + if from != manifest.next_seq { + break; + } + let bytes = self + .store + .read_bytes(&format!("{directory}/{path}")) + .await?; + let decoded = decode_segment(&bytes)?; + if decoded.truncated_tail + || decoded.records.first().map(|record| record.seq) != Some(from) + || decoded.records.last().map(|record| record.seq) != Some(to) + { + return Err(Error::Serialization( + "retained cache segment does not match its filename".to_string(), + )); + } + manifest.segments.push(SegmentDescriptor { + path, + segment_from_seq: from, + segment_to_seq: Some(to), + record_count: decoded.records.len() as u32, + byte_len: bytes.len() as u64, + checksum: Some(sha256_hex(&bytes)), + }); + manifest.next_seq = to + .checked_add(1) + .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?; + } + manifest.validate()?; + Ok(manifest) + } + + /// Parse one deterministic sealed segment filename. + fn sealed_range(name: &str) -> Option<(u64, u64)> { + let range = name + .strip_prefix(SEALED_SEGMENT_FILE_PREFIX)? + .strip_suffix(SEGMENT_FILE_EXTENSION)?; + let (from, to) = range.split_once('-')?; + Some((from.parse().ok()?, to.parse().ok()?)) + } + + /// Encodes account partition identity as script arguments. + fn scope_args(scope: &ScopeKey) -> Vec { + vec![ + Bytes::from(scope.account_id.clone()), + Bytes::from(scope.partition_id.to_string()), + Bytes::from(scope.epoch.to_string()), + ] + } + + /// Returns the leading UTF-8 status from one script result. + fn status<'a>(values: &'a [ScriptValue], operation: &str) -> Result<&'a str> { + Self::string_value(values, 0, operation) + } + + /// Reads one UTF-8 byte result at the requested index. + fn string_value<'a>(values: &'a [ScriptValue], index: usize, field: &str) -> Result<&'a str> { + match values.get(index) { + Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value).map_err(|error| { + Error::Serialization(format!( + "Cache-backed multi-write {field} is not UTF-8: {error}" + )) + }), + other => Err(Error::Serialization(format!( + "Cache-backed multi-write {field} has invalid type: {other:?}" + ))), + } + } + + /// Reads one non-negative integer script result as u64. + fn u64_value(values: &[ScriptValue], index: usize, field: &str) -> Result { + match values.get(index) { + Some(ScriptValue::Integer(value)) => u64::try_from(*value).map_err(|_| { + Error::Serialization(format!("Cache-backed multi-write {field} is negative")) + }), + Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value) + .ok() + .and_then(|value| value.parse().ok()) + .ok_or_else(|| { + Error::Serialization(format!( + "Cache-backed multi-write {field} is not an integer" + )) + }), + other => Err(Error::Serialization(format!( + "Cache-backed multi-write {field} has invalid type: {other:?}" + ))), + } + } + + /// Reads one script integer as u32. + fn u32_value(values: &[ScriptValue], index: usize, field: &str) -> Result { + u32::try_from(Self::u64_value(values, index, field)?).map_err(|_| { + Error::Serialization(format!("Cache-backed multi-write {field} exceeds u32")) + }) + } + + /// Reads one script integer as usize. + fn usize_value(values: &[ScriptValue], index: usize, field: &str) -> Result { + usize::try_from(Self::u64_value(values, index, field)?).map_err(|_| { + Error::Serialization(format!("Cache-backed multi-write {field} exceeds usize")) + }) + } + + /// Deserializes one JSON byte result. + fn json_value(values: &[ScriptValue], index: usize, field: &str) -> Result + where + T: for<'de> Deserialize<'de>, + { + match values.get(index) { + Some(ScriptValue::Bytes(value)) => serde_json::from_slice(value) + .map_err(|error| Error::Serialization(format!("{field}: {error}"))), + other => Err(Error::Serialization(format!( + "Cache-backed multi-write {field} has invalid type: {other:?}" + ))), + } + } + + /// Builds a validated operation error from a script response. + fn script_invalid(values: &[ScriptValue], operation: &str) -> Error { + match Self::string_value(values, 1, operation) { + Ok(message) => Error::invalid_operation(message), + Err(error) => error, + } + } + + /// Builds an error for an unknown script status. + fn unknown_status(operation: &str, status: &str) -> Error { + Error::Serialization(format!( + "Cache-backed multi-write {operation} returned unknown status: {status}" + )) + } + + /// Maps cache runtime failures into existing RAGFS error categories. + fn cache_error(operation: &str, error: CacheError) -> Error { + match error { + CacheError::Timeout(message) => { + Error::Timeout(format!("Cache-backed multi-write {operation}: {message}")) + } + CacheError::Unavailable(message) => { + Error::Network(format!("Cache-backed multi-write {operation}: {message}")) + } + CacheError::Closed => Error::Network(format!( + "Cache-backed multi-write {operation}: runtime is closed" + )), + CacheError::InvalidData(message) => { + Error::Serialization(format!("Cache-backed multi-write {operation}: {message}")) + } + other => Error::internal(format!("Cache-backed multi-write {operation}: {other}")), + } + } +} + +#[async_trait] +impl MultiWriteProvider for CacheProvider { + /// Atomically installs reconstructed state only when cache state is absent. + async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> { + self.validate_scope(scope).await?; + match self.read_manifest(scope).await { + Ok(_) => return Ok(()), + Err(Error::NotFound(_)) => {} + Err(error) => return Err(error), + } + let manifest = self.recovery_manifest(scope, manifest).await?; + let keys = self.keys_for_scope(scope)?; + let mut args = vec![Bytes::from_static(b"bootstrap")]; + args.extend(Self::scope_args(scope)); + args.push(Bytes::from(manifest.version.to_string())); + args.push(Bytes::from(manifest.next_seq.to_string())); + args.push(Bytes::from(serde_json::to_vec(&manifest.segments)?)); + args.push(Bytes::from(serde_json::to_vec(&manifest.backend_states)?)); + let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?; + match Self::status(&values, "bootstrap")? { + "ok" | "exists" => self.read_manifest(scope).await.map(|_| ()), + "invalid" => Err(Self::script_invalid(&values, "bootstrap")), + other => Err(Self::unknown_status("bootstrap", other)), + } + } + + /// Flushes normal events through bounded atomic cache appends. + async fn flush( + &self, + _route_hint: Option<&PartitionContext>, + events: Vec, + ) -> Result { + Self::validate_flush_events(&events)?; + let records = events + .iter() + .map(Self::record) + .collect::>>()?; + let mut offset = 0; + let mut route = None; + let mut continuing = false; + let mut first_seq = None; + let mut last_seq = None; + let mut sealed_segments = Vec::new(); + let owner = Uuid::new_v4().to_string(); + while offset < records.len() { + let (current_route, outcome) = self + .flush_routed_once( + &events, + route.as_ref().map(|route: &PartitionContext| &route.scope), + &records[offset..], + &owner, + continuing, + ) + .await?; + match outcome { + FlushScriptOutcome::SealRequired => { + route.get_or_insert(current_route); + continuing = true; + if let Some(descriptor) = self + .seal_head_with_owner(&route.as_ref().unwrap().scope, Some(&owner), true) + .await? + { + if !sealed_segments + .iter() + .any(|sealed: &SegmentDescriptor| sealed.path == descriptor.path) + { + sealed_segments.push(descriptor); + } + } + } + FlushScriptOutcome::Busy => { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + FlushScriptOutcome::Appended(result) => { + if result.appended == 0 { + return Err(Error::Serialization( + "cache-backed flush appended no records".to_string(), + )); + } + route.get_or_insert(current_route); + first_seq.get_or_insert(result.first_seq); + last_seq = Some(result.last_seq); + offset += result.appended; + if result.needs_seal { + let keep_owner = offset < records.len(); + if let Some(descriptor) = self + .seal_head_with_owner( + &route.as_ref().unwrap().scope, + Some(&owner), + keep_owner, + ) + .await? + { + if !sealed_segments + .iter() + .any(|sealed| sealed.path == descriptor.path) + { + sealed_segments.push(descriptor); + } + } + continuing = keep_owner; + } else if offset < records.len() { + return Err(Error::Serialization( + "cache-backed flush released its owner before completing the batch" + .to_string(), + )); + } + } + } + } + Ok(FlushResult { + scope: route.unwrap().scope, + first_seq: first_seq.unwrap(), + last_seq: last_seq.unwrap(), + sealed_segments, + }) + } + + /// Reads and validates committed cache head records in one half-open range. + async fn read_committed_head( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result> { + let (state, records, _) = self.snapshot(scope).await?; + if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq { + return Err(Error::invalid_operation( + "committed head range is outside the manifest", + )); + } + let head_from = records + .first() + .map(|record| record.seq) + .unwrap_or(state.next_seq); + if from_seq_inclusive < head_from { + return Err(Error::invalid_operation( + "committed head range starts before the current cache head", + )); + } + let expected_len = usize::try_from(to_seq_exclusive - from_seq_inclusive) + .map_err(|_| Error::invalid_operation("committed head range is too large"))?; + let selected = records + .into_iter() + .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive) + .collect::>(); + if selected.len() != expected_len + || selected.iter().enumerate().any(|(index, record)| { + record.seq + != from_seq_inclusive + .checked_add(index as u64) + .unwrap_or(u64::MAX) + }) + { + return Err(Error::Serialization( + "cache committed head range is incomplete".to_string(), + )); + } + Ok(selected) + } + + /// Reads one complete committed range from sealed storage and the cache head. + async fn read_committed_range( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result> { + let (state, head, _) = self.snapshot(scope).await?; + if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq { + return Err(Error::invalid_operation( + "committed range is outside the manifest", + )); + } + let mut records = Vec::new(); + for descriptor in &state.segments { + if descriptor.segment_from_seq >= to_seq_exclusive + || descriptor + .segment_to_seq + .is_some_and(|to| to < from_seq_inclusive) + { + continue; + } + records.extend(read_sealed_segment(&self.store, scope, descriptor).await?); + } + records.extend(head); + select_record_range(records, from_seq_inclusive, to_seq_exclusive) + } + + /// Reads one immutable sealed segment named by a validated descriptor. + async fn read_sealed_segment( + &self, + scope: &ScopeKey, + descriptor: &SegmentDescriptor, + ) -> Result> { + read_sealed_segment(&self.store, scope, descriptor).await + } + + /// Returns validated sealed state plus a derived mutable-head descriptor. + async fn read_manifest(&self, scope: &ScopeKey) -> Result { + let (state, records, _) = self.snapshot(scope).await?; + Self::manifest(state, &records) + } + + /// Appends one caller-scoped marker and seals it before returning. + async fn append_marker( + &self, + scope: &ScopeKey, + event: &PendingEvent, + ) -> Result { + self.validate_scope(scope).await?; + self.validate_marker(scope, event)?; + let record = Self::record(event)?; + let owner = Uuid::new_v4().to_string(); + let mut continuing = false; + loop { + match self + .flush_marker_once(scope, &record, &owner, continuing) + .await? + { + FlushScriptOutcome::SealRequired => { + continuing = true; + self.seal_head_with_owner(scope, Some(&owner), true).await?; + } + FlushScriptOutcome::Busy => { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + FlushScriptOutcome::Appended(result) => { + if result.appended != 1 || !result.needs_seal { + return Err(Error::Serialization( + "cache marker append returned an invalid result".to_string(), + )); + } + self.seal_head_with_owner(scope, Some(&owner), false) + .await?; + if !self.marker_is_sealed(scope, result.first_seq).await? { + return Err(Error::Serialization( + "cache marker is not referenced by a sealed segment".to_string(), + )); + } + return Ok(MarkerPosition { + partition_id: scope.partition_id, + epoch: scope.epoch, + seq: result.first_seq, + }); + } + } + } + } + + /// Advances one backend's monotonic progress atomically through CacheRuntime. + async fn advance_backend_state( + &self, + scope: &ScopeKey, + backend_id: &str, + synced_seq: u64, + ) -> Result<()> { + self.validate_scope(scope).await?; + if backend_id.trim().is_empty() { + return Err(Error::invalid_operation("backend id must not be empty")); + } + let keys = self.keys_for_scope(scope)?; + let mut args = Self::scope_args(scope); + args.push(Bytes::from(backend_id.to_string())); + args.push(Bytes::from(synced_seq.to_string())); + let values = self.execute_array(ADVANCE_SCRIPT_ID, &keys, args).await?; + match Self::status(&values, "advance backend")? { + "ok" => Ok(()), + "not_found" => Err(Error::not_found(keys.state)), + "scope_mismatch" => Err(Error::invalid_operation( + "Cache-backed multi-write state scope does not match", + )), + "invalid" => Err(Self::script_invalid(&values, "advance backend")), + other => Err(Self::unknown_status("advance backend", other)), + } + } + + /// Atomically validates and removes exactly the requested sealed descriptors. + async fn prune_sealed( + &self, + scope: &ScopeKey, + removable_paths: &[String], + ) -> Result> { + let path = self.store.paths().account_manifest(&scope.account_id)?.0; + let lease = self + .store + .pathlock_manager() + .acquire_exact( + &path, + self.store + .pathlock_manager() + .default_lock_timeout() + .max(std::time::Duration::from_secs(30)), + None, + ) + .await?; + let operation = async { + if !self.store.gc_scope_is_safe(scope).await? { + return Ok(Vec::new()); + } + let keys = self.keys_for_scope(scope)?; + let mut args = vec![Bytes::from_static(b"prune")]; + args.extend(Self::scope_args(scope)); + args.push(Bytes::from(serde_json::to_vec(removable_paths)?)); + let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?; + match Self::status(&values, "prune")? { + "ok" => Self::json_value(&values, 1, "pruned descriptors"), + "not_found" => Err(Error::not_found(keys.state)), + "scope_mismatch" => Err(Error::invalid_operation( + "Cache-backed multi-write state scope does not match", + )), + "invalid" => Err(Self::script_invalid(&values, "prune")), + other => Err(Self::unknown_status("prune", other)), + } + } + .await; + let release = self + .store + .pathlock_manager() + .release(&lease) + .await + .map_err(Error::from); + match operation { + Ok(result) => release.map(|()| result), + Err(error) => Err(error), + } + } +} diff --git a/crates/ragfs/src/multibackend/provider/filesystem.rs b/crates/ragfs/src/multibackend/provider/filesystem.rs new file mode 100644 index 0000000000..89840e6329 --- /dev/null +++ b/crates/ragfs/src/multibackend/provider/filesystem.rs @@ -0,0 +1,767 @@ +//! Filesystem-backed V2 multi-write persistence. + +use std::collections::{BTreeSet, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::core::errors::{Error, Result}; +use crate::core::internal_names::is_multiwrite_internal_path; +use crate::lock::{OwnedPathLockLease, PathLockKind, PathLockRequest}; +use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex}; +use crate::multibackend::constants::{ + HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX, + SEGMENT_FILE_EXTENSION, +}; +use crate::multibackend::meta::MetadataStore; +use crate::multibackend::model::{ + FlushResult, MarkerPosition, PartitionContext, PartitionsManifest, PendingEvent, + PendingEventKind, ScopeKey, SegmentDescriptor, SegmentEventType, SegmentManifest, + SegmentRecord, +}; +use crate::multibackend::router::AccountRouter; + +use super::{read_sealed_segment, select_record_range, MultiWriteProvider}; + +/// Persists V2 manifests and segment blobs on the primary filesystem. +pub struct FilesystemProvider { + store: Arc, + router: AccountRouter, +} + +impl FilesystemProvider { + /// Creates a filesystem-backed metadata provider. + pub fn new(store: Arc) -> Self { + Self { + router: AccountRouter::new(store.clone()), + store, + } + } + + /// Validates normal flush input before any route or storage access. + fn validate_flush_events(events: &[PendingEvent]) -> Result<()> { + let first = events + .first() + .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?; + for event in events { + if event.account_id != first.account_id { + return Err(Error::invalid_operation( + "flush events must belong to one account", + )); + } + if is_multiwrite_internal_path(&event.path) { + return Err(Error::invalid_path(format!( + "cannot flush internal path: {}", + event.path + ))); + } + match event.kind { + PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {} + _ => { + return Err(Error::invalid_operation( + "normal flush accepts only write and remove events", + )) + } + } + if event.destination_path.is_some() { + return Err(Error::invalid_operation( + "normal data event cannot have a destination", + )); + } + } + Ok(()) + } + + /// Routes every event and requires one current partition scope. + async fn route_events(&self, events: &[PendingEvent]) -> Result { + let mut selected = None; + for event in events { + let current = self.router.route(&event.account_id, &event.path).await?; + if selected + .as_ref() + .is_some_and(|existing: &PartitionContext| existing.scope != current.scope) + { + return Err(Error::invalid_operation( + "flush events route to different partition scopes", + )); + } + selected.get_or_insert(current); + } + selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event")) + } + + /// Reads and validates the partitions manifest owning one scope. + async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> { + let path = self.store.paths().account_manifest(&scope.account_id)?.0; + let manifest: PartitionsManifest = self.store.read_json(&path).await?; + manifest.validate()?; + scope.validate_against_manifest(&manifest) + } + + /// Reads and validates one segment manifest without acquiring locks. + async fn load_manifest(&self, scope: &ScopeKey) -> Result { + self.validate_scope(scope).await?; + let path = self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0; + let manifest: SegmentManifest = self.store.read_json(&path).await?; + manifest.validate()?; + Ok(manifest) + } + + /// Returns the current mutable head descriptor. + fn head(manifest: &SegmentManifest) -> Option<&SegmentDescriptor> { + manifest + .segments + .last() + .filter(|segment| segment.segment_to_seq.is_none()) + } + + /// Returns the deterministic mutable head filename. + fn head_name(from_seq: u64) -> String { + format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}") + } + + /// Returns the deterministic immutable sealed filename. + fn sealed_name(from_seq: u64, to_seq: u64) -> String { + format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}") + } + + /// Returns one logical segment blob path. + fn segment_path(&self, scope: &ScopeKey, name: &str) -> Result { + let directory = self + .store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + Ok(format!("{directory}/{name}")) + } + + /// Builds the complete Exact-lock set for one append candidate. + fn append_lock_requests( + &self, + scope: &ScopeKey, + manifest: &SegmentManifest, + event_count: usize, + force_final_seal: bool, + ) -> Result> { + let mut paths = BTreeSet::from([self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0]); + paths.insert(self.store.paths().account_manifest(&scope.account_id)?.0); + let mut remaining = event_count; + let mut next_seq = manifest.next_seq; + if next_seq.checked_add(event_count as u64).is_none() { + return Err(Error::invalid_operation("multi-write sequence overflow")); + } + let mut count = 0; + let mut segment_from_seq = next_seq; + if let Some(head) = Self::head(manifest) { + paths.insert(self.segment_path(scope, &head.path)?); + count = head.record_count as usize; + segment_from_seq = head.segment_from_seq; + } + + if count == MAX_SEGMENT_RECORDS { + paths.insert( + self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?, + ); + count = 0; + segment_from_seq = next_seq; + } + while remaining > 0 { + let appended = remaining.min(MAX_SEGMENT_RECORDS - count); + remaining -= appended; + next_seq = next_seq + .checked_add(appended as u64) + .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?; + count += appended; + if count == MAX_SEGMENT_RECORDS || (force_final_seal && remaining == 0) { + paths.insert( + self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?, + ); + count = 0; + segment_from_seq = next_seq; + } + } + if count > 0 { + let name = if force_final_seal { + Self::sealed_name(segment_from_seq, next_seq - 1) + } else { + Self::head_name(segment_from_seq) + }; + paths.insert(self.segment_path(scope, &name)?); + } + Ok(paths + .into_iter() + .map(|path| PathLockRequest { + path, + kind: PathLockKind::Exact, + }) + .collect()) + } + + /// Builds the Exact-lock set for a manifest-only state mutation. + fn manifest_lock_requests(&self, scope: &ScopeKey) -> Result> { + let mut paths = vec![PathLockRequest { + path: self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0, + kind: PathLockKind::Exact, + }]; + paths.push(PathLockRequest { + path: self.store.paths().account_manifest(&scope.account_id)?.0, + kind: PathLockKind::Exact, + }); + Ok(paths) + } + + /// Acquires one all-or-nothing Exact-lock batch. + async fn acquire(&self, requests: &[PathLockRequest]) -> Result { + let manager = self.store.pathlock_manager(); + Ok(manager + .acquire_batch(requests, std::time::Duration::from_secs(5), None) + .await?) + } + + /// Releases a lock lease while preserving any operation failure. + async fn release(&self, lease: &OwnedPathLockLease, operation: Result) -> Result { + let release = self + .store + .pathlock_manager() + .release(lease) + .await + .map_err(Error::from); + merge_operation_and_release(operation, release) + } + + /// Reads and validates only the committed prefix of the current head. + async fn committed_head( + &self, + scope: &ScopeKey, + manifest: &SegmentManifest, + ) -> Result> { + let Some(head) = Self::head(manifest) else { + return Ok(Vec::new()); + }; + if head.path != Self::head_name(head.segment_from_seq) { + return Err(Error::Serialization( + "head segment path is not deterministic".to_string(), + )); + } + let bytes = self + .store + .read_bytes(&self.segment_path(scope, &head.path)?) + .await?; + let committed_len = usize::try_from(head.byte_len) + .map_err(|_| Error::Serialization("head byte length is too large".to_string()))?; + if bytes.len() < committed_len { + return Err(Error::Serialization( + "head segment is shorter than its committed byte length".to_string(), + )); + } + let decoded = decode_segment(&bytes[..committed_len])?; + if decoded.truncated_tail + || decoded.valid_bytes != head.byte_len + || decoded.records.len() != head.record_count as usize + { + return Err(Error::Serialization( + "head segment committed prefix does not match its descriptor".to_string(), + )); + } + for (index, record) in decoded.records.iter().enumerate() { + let expected = head + .segment_from_seq + .checked_add(index as u64) + .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?; + if record.seq != expected { + return Err(Error::Serialization( + "head segment contains a sequence gap".to_string(), + )); + } + } + Ok(decoded.records) + } + + /// Converts one pending event into its persisted segment record. + fn record(event: &PendingEvent, seq: u64) -> Result { + let event_type = match event.kind { + PendingEventKind::Data(kind) => kind, + PendingEventKind::DeleteAccount => { + return Err(Error::invalid_operation( + "account deletion is not a segment record", + )) + } + }; + let marker = matches!( + event_type, + SegmentEventType::RemoveTree + | SegmentEventType::MoveTree + | SegmentEventType::DeleteScope + ); + let record = SegmentRecord { + seq, + event_type, + path: event.path.clone(), + op_id: marker.then_some(event.operation_id), + destination_path: event.destination_path.clone(), + }; + record.validate()?; + Ok(record) + } + + /// Persists one immutable sealed segment and returns its descriptor. + async fn persist_sealed( + &self, + scope: &ScopeKey, + manifest: &SegmentManifest, + records: &[SegmentRecord], + ) -> Result { + let first = records + .first() + .ok_or_else(|| Error::invalid_operation("cannot seal an empty segment"))? + .seq; + let last = records.last().unwrap().seq; + let bytes = encode_segment(records)?; + let name = Self::sealed_name(first, last); + let path = self.segment_path(scope, &name)?; + if !manifest.segments.iter().any(|segment| segment.path == name) { + self.store.remove_file(&path).await?; + } + self.store.write_immutable(&path, &bytes).await?; + Ok(SegmentDescriptor { + path: name, + segment_from_seq: first, + segment_to_seq: Some(last), + record_count: records.len() as u32, + byte_len: bytes.len() as u64, + checksum: Some(sha256_hex(&bytes)), + }) + } + + /// Publishes one mutable head and returns its descriptor. + async fn persist_head( + &self, + scope: &ScopeKey, + records: &[SegmentRecord], + ) -> Result { + let first = records + .first() + .ok_or_else(|| Error::invalid_operation("cannot publish an empty head"))? + .seq; + let bytes = encode_segment(records)?; + let name = Self::head_name(first); + self.store + .write_bytes(&self.segment_path(scope, &name)?, &bytes) + .await?; + Ok(SegmentDescriptor { + path: name, + segment_from_seq: first, + segment_to_seq: None, + record_count: records.len() as u32, + byte_len: bytes.len() as u64, + checksum: None, + }) + } + + /// Appends records to a locked manifest and publishes blobs before the manifest. + async fn commit_events( + &self, + scope: &ScopeKey, + mut manifest: SegmentManifest, + events: &[PendingEvent], + force_final_seal: bool, + ) -> Result<(SegmentManifest, Vec)> { + let mut records = self.committed_head(scope, &manifest).await?; + let mut sealed = Vec::new(); + if records.len() == MAX_SEGMENT_RECORDS { + manifest.segments.pop(); + let descriptor = self.persist_sealed(scope, &manifest, &records).await?; + manifest.segments.push(descriptor.clone()); + sealed.push(descriptor); + records.clear(); + } + + for (index, event) in events.iter().enumerate() { + let seq = manifest.next_seq; + manifest.next_seq = manifest + .next_seq + .checked_add(1) + .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?; + records.push(Self::record(event, seq)?); + let seal_now = records.len() == MAX_SEGMENT_RECORDS + || (force_final_seal && index + 1 == events.len()); + if seal_now { + if Self::head(&manifest).is_some() { + manifest.segments.pop(); + } + let descriptor = self.persist_sealed(scope, &manifest, &records).await?; + manifest.segments.push(descriptor.clone()); + sealed.push(descriptor); + records.clear(); + } + } + + if !records.is_empty() { + let descriptor = self.persist_head(scope, &records).await?; + if Self::head(&manifest).is_some() { + manifest.segments.pop(); + } + manifest.segments.push(descriptor); + } + manifest.validate()?; + let path = self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0; + self.store + .publish_json(&path, &manifest, SegmentManifest::validate) + .await?; + Ok((manifest, sealed)) + } + + /// Returns whether the locked append snapshot still matches its candidate. + fn append_snapshot_matches( + candidate_scope: &ScopeKey, + current_scope: &ScopeKey, + candidate_manifest: &SegmentManifest, + current_manifest: &SegmentManifest, + candidate_requests: &[PathLockRequest], + current_requests: &[PathLockRequest], + ) -> bool { + candidate_scope == current_scope + && candidate_manifest == current_manifest + && candidate_requests == current_requests + } + + /// Validates one marker against its caller-selected scope. + fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> { + if event.account_id != scope.account_id { + return Err(Error::invalid_operation( + "marker account does not match supplied scope", + )); + } + match event.kind { + PendingEventKind::Data( + SegmentEventType::RemoveTree + | SegmentEventType::MoveTree + | SegmentEventType::DeleteScope, + ) => {} + _ => { + return Err(Error::invalid_operation( + "append_marker accepts only directory markers", + )) + } + } + if is_multiwrite_internal_path(&event.path) { + return Err(Error::invalid_path(format!( + "cannot append marker for internal path: {}", + event.path + ))); + } + if !matches!( + event.kind, + PendingEventKind::Data(SegmentEventType::DeleteScope) + ) { + self.store + .paths() + .backend_path(&scope.account_id, &event.path)?; + } + if let Some(destination) = &event.destination_path { + if is_multiwrite_internal_path(destination) { + return Err(Error::invalid_path(format!( + "cannot append marker for internal destination: {destination}" + ))); + } + self.store + .paths() + .backend_path(&scope.account_id, destination)?; + } + Self::record(event, 1).map(|_| ()) + } +} + +#[async_trait] +impl MultiWriteProvider for FilesystemProvider { + /// Confirms that filesystem state exists for the account partition. + async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> { + manifest.validate()?; + self.load_manifest(scope).await.map(|_| ()) + } + + /// Flushes normal events after validating their current partition route. + async fn flush( + &self, + _route_hint: Option<&PartitionContext>, + events: Vec, + ) -> Result { + Self::validate_flush_events(&events)?; + loop { + let candidate = self.route_events(&events).await?; + let candidate_manifest = self.load_manifest(&candidate.scope).await?; + let requests = self.append_lock_requests( + &candidate.scope, + &candidate_manifest, + events.len(), + false, + )?; + let lease = self.acquire(&requests).await?; + let operation = async { + let current = self.route_events(&events).await?; + let manifest = self.load_manifest(¤t.scope).await?; + let current_requests = + self.append_lock_requests(¤t.scope, &manifest, events.len(), false)?; + if !Self::append_snapshot_matches( + &candidate.scope, + ¤t.scope, + &candidate_manifest, + &manifest, + &requests, + ¤t_requests, + ) { + return Ok(None); + } + let first_seq = manifest.next_seq; + let (_, sealed_segments) = self + .commit_events(¤t.scope, manifest, &events, false) + .await?; + let last_seq = first_seq + .checked_add(events.len() as u64 - 1) + .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?; + Ok(Some(FlushResult { + scope: current.scope, + first_seq, + last_seq, + sealed_segments, + })) + } + .await; + if let Some(result) = self.release(&lease, operation).await? { + return Ok(result); + } + } + } + + /// Reads committed head records in the requested half-open sequence range. + async fn read_committed_head( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result> { + let manifest = self.load_manifest(scope).await?; + if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq { + return Err(Error::invalid_operation( + "committed head range is outside the manifest", + )); + } + let records = self.committed_head(scope, &manifest).await?; + Ok(records + .into_iter() + .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive) + .collect()) + } + + /// Reads one complete committed range from sealed segments and the current head. + async fn read_committed_range( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result> { + let manifest = self.load_manifest(scope).await?; + if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq { + return Err(Error::invalid_operation( + "committed range is outside the manifest", + )); + } + let mut records = Vec::new(); + for descriptor in &manifest.segments { + if descriptor.segment_from_seq >= to_seq_exclusive + || descriptor + .segment_to_seq + .is_some_and(|to| to < from_seq_inclusive) + { + continue; + } + if descriptor.segment_to_seq.is_some() { + records.extend(read_sealed_segment(&self.store, scope, descriptor).await?); + } else { + records.extend(self.committed_head(scope, &manifest).await?); + } + } + select_record_range(records, from_seq_inclusive, to_seq_exclusive) + } + + /// Reads one immutable sealed segment named by a validated descriptor. + async fn read_sealed_segment( + &self, + scope: &ScopeKey, + descriptor: &SegmentDescriptor, + ) -> Result> { + read_sealed_segment(&self.store, scope, descriptor).await + } + + /// Returns the current segment state for one scope. + async fn read_manifest(&self, scope: &ScopeKey) -> Result { + self.load_manifest(scope).await + } + + /// Appends and seals one directory marker in the caller-selected scope. + async fn append_marker( + &self, + scope: &ScopeKey, + event: &PendingEvent, + ) -> Result { + self.validate_marker(scope, event)?; + loop { + let candidate_manifest = self.load_manifest(scope).await?; + let requests = self.append_lock_requests(scope, &candidate_manifest, 1, true)?; + let lease = self.acquire(&requests).await?; + let operation = async { + let manifest = self.load_manifest(scope).await?; + let current_requests = self.append_lock_requests(scope, &manifest, 1, true)?; + if !Self::append_snapshot_matches( + scope, + scope, + &candidate_manifest, + &manifest, + &requests, + ¤t_requests, + ) { + return Ok(None); + } + let seq = manifest.next_seq; + self.commit_events(scope, manifest, std::slice::from_ref(event), true) + .await?; + Ok(Some(MarkerPosition { + partition_id: scope.partition_id, + epoch: scope.epoch, + seq, + })) + } + .await; + if let Some(position) = self.release(&lease, operation).await? { + return Ok(position); + } + } + } + + /// Advances one backup's progress while its catch-up lease is held. + async fn advance_backend_state( + &self, + scope: &ScopeKey, + backend_id: &str, + synced_seq: u64, + ) -> Result<()> { + let requests = self.manifest_lock_requests(scope)?; + let lease = self.acquire(&requests).await?; + let operation = async { + let mut manifest = self.load_manifest(scope).await?; + let state = manifest.backend_states.get_mut(backend_id).ok_or_else(|| { + Error::invalid_operation(format!("unknown backend id: {backend_id}")) + })?; + if synced_seq != 0 && synced_seq < state.synced_seq { + return Err(Error::invalid_operation("backend progress cannot regress")); + } + if synced_seq >= manifest.next_seq { + return Err(Error::invalid_operation( + "backend progress must remain below next_seq", + )); + } + state.synced_seq = synced_seq; + let path = self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0; + self.store + .publish_json(&path, &manifest, SegmentManifest::validate) + .await + } + .await; + self.release(&lease, operation).await + } + + /// Atomically removes selected sealed descriptors and returns the removed set. + async fn prune_sealed( + &self, + scope: &ScopeKey, + removable_paths: &[String], + ) -> Result> { + let requested = removable_paths.iter().collect::>(); + if requested.len() != removable_paths.len() { + return Err(Error::invalid_operation( + "prune paths must not contain duplicates", + )); + } + let requests = self.manifest_lock_requests(scope)?; + let lease = self.acquire(&requests).await?; + let operation = async { + if !self.store.gc_scope_is_safe(scope).await? { + return Ok(Vec::new()); + } + let mut manifest = self.load_manifest(scope).await?; + for path in removable_paths { + let descriptor = manifest + .segments + .iter() + .find(|segment| segment.path == *path) + .ok_or_else(|| { + Error::invalid_operation(format!( + "sealed segment is not referenced: {path}" + )) + })?; + if descriptor.segment_to_seq.is_none() { + return Err(Error::invalid_operation(format!( + "cannot prune mutable head: {path}" + ))); + } + } + let mut removed = Vec::new(); + manifest.segments.retain(|segment| { + if requested.contains(&segment.path) { + removed.push(segment.clone()); + false + } else { + true + } + }); + manifest.validate()?; + let path = self + .store + .paths() + .segment_manifest(&scope.account_id, scope.partition_id)? + .0; + self.store + .publish_json(&path, &manifest, SegmentManifest::validate) + .await?; + Ok(removed) + } + .await; + self.release(&lease, operation).await + } +} + +/// Preserves an operation error when lock release also fails. +fn merge_operation_and_release(operation: Result, release: Result<()>) -> Result { + match (operation, release) { + (Ok(value), Ok(())) => Ok(value), + (Ok(_), Err(error)) => Err(error), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(release_error)) => Err(Error::internal(format!( + "{error}; lock release failed: {release_error}" + ))), + } +} + +#[cfg(test)] +#[path = "tests/test_filesystem.rs"] +mod tests; diff --git a/crates/ragfs/src/multibackend/provider/mod.rs b/crates/ragfs/src/multibackend/provider/mod.rs new file mode 100644 index 0000000000..1f99117287 --- /dev/null +++ b/crates/ragfs/src/multibackend/provider/mod.rs @@ -0,0 +1,179 @@ +//! Persistence providers for V2 multi-write state. + +use async_trait::async_trait; + +use crate::core::errors::{Error, Result}; +use crate::multibackend::codec::{decode_segment, sha256_hex}; +use crate::multibackend::meta::MetadataStore; +use crate::multibackend::model::{ + FlushResult, MarkerPosition, PartitionContext, PartitionState, PartitionsManifest, + PendingEvent, ScopeKey, SegmentDescriptor, SegmentManifest, SegmentRecord, +}; + +mod filesystem; +#[cfg(feature = "cache")] +mod cache; + +pub use filesystem::FilesystemProvider; +#[cfg(feature = "cache")] +pub use cache::CacheProvider; + +/// Persists and reads partitioned multi-write segment state. +#[async_trait] +pub trait MultiWriteProvider: Send + Sync { + /// Creates missing provider state without replacing an existing manifest. + async fn bootstrap_manifest( + &self, + scope: &ScopeKey, + manifest: SegmentManifest, + ) -> Result<()>; + + /// Flushes normal events after validating their current partition route. + async fn flush( + &self, + route_hint: Option<&PartitionContext>, + events: Vec, + ) -> Result; + + /// Reads committed head records in the requested half-open sequence range. + async fn read_committed_head( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result>; + + /// Reads committed records across sealed segments and the current head. + async fn read_committed_range( + &self, + scope: &ScopeKey, + from_seq_inclusive: u64, + to_seq_exclusive: u64, + ) -> Result>; + + /// Reads one immutable sealed segment named by a validated descriptor. + async fn read_sealed_segment( + &self, + scope: &ScopeKey, + descriptor: &SegmentDescriptor, + ) -> Result>; + + /// Returns the current segment state for one scope. + async fn read_manifest(&self, scope: &ScopeKey) -> Result; + + /// Appends and seals one directory marker in the caller-selected scope. + async fn append_marker( + &self, + scope: &ScopeKey, + event: &PendingEvent, + ) -> Result; + + /// Advances one backup's monotonic progress while its catch-up lease is held. + async fn advance_backend_state( + &self, + scope: &ScopeKey, + backend_id: &str, + synced_seq: u64, + ) -> Result<()>; + + /// Atomically removes selected sealed descriptors and returns the removed set. + async fn prune_sealed( + &self, + scope: &ScopeKey, + removable_paths: &[String], + ) -> Result>; + +} + +/// Bootstrap every stable provider partition from its raw-primary manifest. +pub(crate) async fn bootstrap_account( + provider: &dyn MultiWriteProvider, + store: &MetadataStore, + account_id: &str, + partitions: &PartitionsManifest, +) -> Result<()> { + for (&partition_id, partition) in &partitions.partitions { + if partition.state != PartitionState::Stable { + continue; + } + let scope = ScopeKey { + account_id: account_id.to_string(), + partition_id, + epoch: partitions.epoch, + }; + let path = store.paths().segment_manifest(account_id, partition_id)?.0; + provider.bootstrap_manifest(&scope, store.read_json(&path).await?).await?; + } + Ok(()) +} + +/// Read and validate one immutable segment through the shared metadata store. +async fn read_sealed_segment( + store: &MetadataStore, + scope: &ScopeKey, + descriptor: &SegmentDescriptor, +) -> Result> { + let directory = store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + let bytes = store + .read_bytes(&format!("{directory}/{}", descriptor.path)) + .await?; + if bytes.len() as u64 != descriptor.byte_len + || sha256_hex(&bytes) != descriptor.checksum.as_deref().unwrap_or_default() + { + return Err(Error::Serialization( + "sealed segment bytes do not match descriptor".to_string(), + )); + } + let decoded = decode_segment(&bytes)?; + if decoded.truncated_tail + || decoded.valid_bytes != descriptor.byte_len + || decoded.records.len() != descriptor.record_count as usize + || decoded.records.iter().enumerate().any(|(index, record)| { + record.seq + != descriptor + .segment_from_seq + .checked_add(index as u64) + .unwrap_or(u64::MAX) + }) + { + return Err(Error::Serialization( + "sealed segment records do not match descriptor".to_string(), + )); + } + Ok(decoded.records) +} + +/// Select and validate one complete half-open sequence range. +fn select_record_range( + records: impl IntoIterator, + from_seq_inclusive: u64, + to_seq_exclusive: u64, +) -> Result> { + let range_len = to_seq_exclusive + .checked_sub(from_seq_inclusive) + .ok_or_else(|| Error::invalid_operation("committed range is reversed"))?; + let expected_len = usize::try_from(range_len) + .map_err(|_| Error::invalid_operation("committed range is too large"))?; + let selected = records + .into_iter() + .filter(|record| { + record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive + }) + .collect::>(); + if selected.len() != expected_len + || selected.iter().enumerate().any(|(index, record)| { + record.seq + != from_seq_inclusive + .checked_add(index as u64) + .unwrap_or(u64::MAX) + }) + { + return Err(Error::Serialization( + "committed record range is incomplete".to_string(), + )); + } + Ok(selected) +} diff --git a/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs new file mode 100644 index 0000000000..636eb4381a --- /dev/null +++ b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs @@ -0,0 +1,72 @@ +use super::*; +use crate::core::FileSystem; +use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager}; +use crate::plugins::memfs::MemFileSystem; + +/// Verifies flush publishes a contiguous range that committed reads can recover. +#[tokio::test] +async fn flush_publishes_a_complete_committed_range() { + let raw_primary: Arc = Arc::new(MemFileSystem::new()); + let pathlock_manager = Arc::new(PathLockManager::new( + raw_primary.clone(), + Arc::new(MemoryPathLockProvider::new()), + PathLockConfig::default(), + )); + let store = Arc::new(MetadataStore::new(raw_primary, pathlock_manager, "/local").unwrap()); + let partitions = store.initialize_account("account", 1, &[]).await.unwrap(); + let provider = FilesystemProvider::new(store); + let scope = ScopeKey { + account_id: "account".to_string(), + partition_id: 0, + epoch: partitions.epoch, + }; + + let result = provider + .flush( + None, + vec![ + PendingEvent { + operation_id: uuid::Uuid::from_u128(1), + account_id: "account".to_string(), + kind: PendingEventKind::Data(SegmentEventType::Write), + path: "/local/account/first.txt".to_string(), + destination_path: None, + route_hint: None, + }, + PendingEvent { + operation_id: uuid::Uuid::from_u128(2), + account_id: "account".to_string(), + kind: PendingEventKind::Data(SegmentEventType::Remove), + path: "/local/account/second.txt".to_string(), + destination_path: None, + route_hint: None, + }, + ], + ) + .await + .unwrap(); + + assert_eq!(result.scope, scope); + assert_eq!((result.first_seq, result.last_seq), (1, 2)); + let records = provider.read_committed_range(&scope, 1, 3).await.unwrap(); + assert_eq!( + records, + vec![ + SegmentRecord { + seq: 1, + event_type: SegmentEventType::Write, + path: "/local/account/first.txt".to_string(), + op_id: None, + destination_path: None, + }, + SegmentRecord { + seq: 2, + event_type: SegmentEventType::Remove, + path: "/local/account/second.txt".to_string(), + op_id: None, + destination_path: None, + }, + ] + ); + assert!(provider.read_committed_range(&scope, 2, 4).await.is_err()); +} diff --git a/crates/ragfs/src/multibackend/router.rs b/crates/ragfs/src/multibackend/router.rs new file mode 100644 index 0000000000..209df71f21 --- /dev/null +++ b/crates/ragfs/src/multibackend/router.rs @@ -0,0 +1,274 @@ +//! Logical path mapping and account partition routing. + +use std::collections::HashSet; +use std::sync::Arc; + +use xxhash_rust::xxh3::xxh3_64; + +use crate::core::errors::{Error, Result}; +use crate::multibackend::constants::{ + CHECKPOINTS_DIR, MANIFEST_FILE, MULTIWRITE_MOUNT_PREFIX, MULTIWRITE_PROTOCOL_FILE, + PARTITIONS_DIR, SEGMENTS_DIR, SYSTEM_DIR, VBUCKETS, +}; +use crate::multibackend::meta::MetadataStore; +use crate::multibackend::model::{PartitionContext, PartitionsManifest, RouteEntry, ScopeKey}; + +/// Maps canonical logical metadata paths to raw backend paths. +#[derive(Debug, Clone)] +pub struct MultiWritePaths { + mount_prefix: String, +} + +impl MultiWritePaths { + /// Create a path mapper for one canonical absolute mount prefix. + pub fn new(mount_prefix: &str) -> Result { + validate_canonical_path(mount_prefix)?; + if mount_prefix != MULTIWRITE_MOUNT_PREFIX { + return Err(Error::invalid_path(format!( + "unsupported multi-write mount prefix: {mount_prefix}" + ))); + } + Ok(Self { + mount_prefix: mount_prefix.to_string(), + }) + } + + /// Return logical and backend paths for the mount protocol manifest. + pub fn mount_protocol(&self) -> (String, String) { + self.pair(&format!( + "{}/{SYSTEM_DIR}/{MULTIWRITE_PROTOCOL_FILE}", + self.mount_prefix + )) + } + + /// Return logical and backend paths for an account partitions directory. + pub fn account_partitions(&self, account_id: &str) -> Result<(String, String)> { + validate_account(account_id)?; + Ok(self.pair(&format!( + "{}/{account_id}/{SYSTEM_DIR}/{PARTITIONS_DIR}", + self.mount_prefix + ))) + } + + /// Return logical and backend paths for an account partitions manifest. + pub fn account_manifest(&self, account_id: &str) -> Result<(String, String)> { + let (logical, _) = self.account_partitions(account_id)?; + Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}"))) + } + + /// Return logical and backend paths for one partition manifest. + pub fn partition_manifest( + &self, + account_id: &str, + partition_id: u32, + ) -> Result<(String, String)> { + let (logical, _) = self.account_partitions(account_id)?; + Ok(self.pair(&format!("{logical}/{partition_id}/{MANIFEST_FILE}"))) + } + + /// Return logical and backend paths for one partition segments directory. + pub fn segments_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> { + let (logical, _) = self.account_partitions(account_id)?; + Ok(self.pair(&format!("{logical}/{partition_id}/{SEGMENTS_DIR}"))) + } + + /// Return logical and backend paths for one segment manifest. + pub fn segment_manifest( + &self, + account_id: &str, + partition_id: u32, + ) -> Result<(String, String)> { + let (logical, _) = self.segments_dir(account_id, partition_id)?; + Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}"))) + } + + /// Return logical and backend paths for one partition checkpoints directory. + pub fn checkpoints_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> { + let (logical, _) = self.account_partitions(account_id)?; + Ok(self.pair(&format!("{logical}/{partition_id}/{CHECKPOINTS_DIR}"))) + } + + /// Return logical and backend paths for one checkpoints manifest. + pub fn checkpoints_manifest( + &self, + account_id: &str, + partition_id: u32, + ) -> Result<(String, String)> { + let (logical, _) = self.checkpoints_dir(account_id, partition_id)?; + Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}"))) + } + + /// Convert a canonical logical account path to its raw backend path. + pub fn backend_path(&self, account_id: &str, logical_path: &str) -> Result { + validate_account(account_id)?; + let account_root = format!("{}/{account_id}", self.mount_prefix); + if logical_path != account_root + && !logical_path + .strip_prefix(&account_root) + .is_some_and(|suffix| suffix.starts_with('/')) + { + return Err(Error::invalid_path(format!( + "logical path does not belong to account '{account_id}': {logical_path}" + ))); + } + self.raw_backend_path(logical_path) + } + + /// Convert any canonical logical mount path to its raw backend path. + pub fn raw_backend_path(&self, logical_path: &str) -> Result { + validate_canonical_path(logical_path)?; + let suffix = logical_path + .strip_prefix(&self.mount_prefix) + .filter(|suffix| suffix.starts_with('/')) + .ok_or_else(|| { + Error::invalid_path(format!( + "logical path is outside mount '{}': {logical_path}", + self.mount_prefix + )) + })?; + Ok(suffix.to_string()) + } + + /// Hash a canonical full logical account path into its fixed vbucket. + pub fn vbucket(&self, account_id: &str, logical_path: &str) -> Result { + self.backend_path(account_id, logical_path)?; + Ok((xxh3_64(logical_path.as_bytes()) & u64::from(VBUCKETS - 1)) as u32) + } + + /// Pair one validated logical path with its raw backend form. + fn pair(&self, logical_path: &str) -> (String, String) { + ( + logical_path.to_string(), + logical_path[self.mount_prefix.len()..].to_string(), + ) + } +} + +/// Routes account paths through the current persisted partitions manifest. +pub struct AccountRouter { + store: Arc, +} + +impl AccountRouter { + /// Create an account router backed by the supplied metadata store. + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Route one canonical logical account path to its partition context. + pub async fn route(&self, account_id: &str, logical_path: &str) -> Result { + let bucket = self.store.paths().vbucket(account_id, logical_path)?; + let manifest_path = self.store.paths().account_manifest(account_id)?.0; + let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?; + manifest.validate()?; + let index = manifest.routes.partition_point(|route| route.end < bucket); + let route = manifest + .routes + .get(index) + .ok_or_else(|| Error::Serialization("vbucket route is missing".to_string()))?; + let segment_dir = self + .store + .paths() + .segments_dir(account_id, route.partition)? + .1; + Ok(PartitionContext { + scope: ScopeKey { + account_id: account_id.to_string(), + partition_id: route.partition, + epoch: manifest.epoch, + }, + segment_dir, + }) + } +} + +/// Validate compressed complete routes against the known partition identifiers. +pub fn validate_routes( + routes: &[RouteEntry], + partition_ids: impl IntoIterator, +) -> Result<()> { + let partition_ids = partition_ids.into_iter().collect::>(); + let mut next = 0; + let mut previous_owner = None; + for route in routes { + route.validate()?; + if route.start != next { + return Err(Error::Serialization( + "routes are not contiguous".to_string(), + )); + } + if !partition_ids.contains(&route.partition) { + return Err(Error::Serialization("route owner is unknown".to_string())); + } + if previous_owner == Some(route.partition) { + return Err(Error::Serialization( + "adjacent routes have the same owner".to_string(), + )); + } + next = route.end + 1; + previous_owner = Some(route.partition); + } + if next != VBUCKETS { + return Err(Error::Serialization( + "routes do not cover all vbuckets".to_string(), + )); + } + Ok(()) +} + +/// Build deterministic balanced routes for the requested initial partition count. +pub fn build_initial_routes(partition_count: u32) -> Result> { + if !(1..=VBUCKETS).contains(&partition_count) { + return Err(Error::Serialization(format!( + "partition count must be between 1 and {VBUCKETS}" + ))); + } + let owners = (0..VBUCKETS) + .map(|bucket| { ((u64::from(bucket) * u64::from(partition_count)) / u64::from(VBUCKETS)) as u32}) + .collect::>(); + let routes = compress_owners(&owners); + validate_routes(&routes, 0..partition_count)?; + Ok(routes) +} + +/// Compress a complete owner table into inclusive route ranges. +pub(crate) fn compress_owners(owners: &[u32]) -> Vec { + let mut routes = Vec::new(); + let mut start = 0; + for index in 1..=owners.len() { + if index == owners.len() || owners[index] != owners[start] { + routes.push(RouteEntry { + start: start as u32, + end: index as u32 - 1, + partition: owners[start], + }); + start = index; + } + } + routes +} + +/// Validate an account identifier as one canonical path component. +fn validate_account(account_id: &str) -> Result<()> { + if account_id.is_empty() || account_id == "." || account_id == ".." || account_id.contains('/') + { + return Err(Error::invalid_path(format!( + "invalid account identifier: {account_id}" + ))); + } + Ok(()) +} + +/// Validate an absolute path without empty, dot, or trailing components. +fn validate_canonical_path(path: &str) -> Result<()> { + if !path.starts_with('/') + || path.len() <= 1 + || path.ends_with('/') + || path.split('/').skip(1).any(|part| { + part.is_empty() || part == "." || part == ".." + }) + { + return Err(Error::invalid_path(format!("non-canonical path: {path}"))); + } + Ok(()) +} diff --git a/crates/ragfs/src/multibackend/runtime.rs b/crates/ragfs/src/multibackend/runtime.rs new file mode 100644 index 0000000000..c9f2e16b85 --- /dev/null +++ b/crates/ragfs/src/multibackend/runtime.rs @@ -0,0 +1,755 @@ +//! Foreground event queue and metadata flush worker for V2 multi-write. + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::Duration; + +use tokio::sync::{mpsc, watch}; +use tokio::task::JoinHandle; +use tracing::{debug, error}; + +use crate::core::context::{FsContextInner, FS_CTX}; +use crate::core::errors::{Error, Result}; +use crate::core::filesystem::FileSystem; +use crate::multibackend::catch_up::{CatchUpTarget, CatchUpWorker}; +use crate::multibackend::checkpoint::{CheckpointReader, CheckpointWorker}; +use crate::multibackend::codec::{decode_segment, sha256_hex}; +use crate::multibackend::constants::SYSTEM_DIR; +use crate::multibackend::meta::{MetadataStore, MultiWriteWorker}; +use crate::multibackend::model::{ + DirectoryEvent, DirectoryOperation, MarkerPosition, PartitionContext, PartitionState, + PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentEventType, +}; +use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider}; +use crate::multibackend::router::AccountRouter; + +/// Read active account ids from the encrypted or plain primary registry. +pub(crate) async fn read_active_accounts(primary: &dyn FileSystem) -> Result> { + let context = Arc::new( + FsContextInner::new(SYSTEM_DIR) + .with_bypass_cache(true) + .with_auto_pathlock_disabled(), + ); + let bytes = match FS_CTX + .scope(context, primary.read("/_system/accounts.json", 0, 0)) + .await + { + Ok(bytes) => bytes, + Err(Error::NotFound(_)) => return Ok(HashSet::new()), + Err(error) => return Err(error), + }; + let value: serde_json::Value = serde_json::from_slice(&bytes)?; + let accounts = value + .get("accounts") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?; + Ok(accounts + .iter() + .filter(|(_, account)| { + account + .get("deletion") + .is_none_or(serde_json::Value::is_null) + }) + .map(|(account_id, _)| account_id.clone()) + .collect()) +} + +/// Owns the non-blocking foreground sender and its background flush worker. +pub struct MultiWriteRuntime { + sender: StdMutex>>, + activation: watch::Sender>, + import_worker: StdMutex>>>, + store: Arc, + catch_up_cancelled: watch::Sender, + catch_up_wake: Arc, + running: Arc, + pending_events: Arc, +} + +impl MultiWriteRuntime { + /// Assemble runtime tasks without allowing background work to start. + pub(crate) async fn prepare( + store: Arc, + provider: Arc, + initial_partitions: u32, + backup_names: Vec, + primary: Option>, + targets: Vec, + checkpoint_interval: Duration, + ) -> Self { + let (sender, receiver) = mpsc::unbounded_channel(); + let running = Arc::new(AtomicBool::new(true)); + let worker_running = running.clone(); + let pending_events = Arc::new(AtomicUsize::new(0)); + let worker_pending_events = pending_events.clone(); + let (activation, _) = watch::channel(None); + let (catch_up_cancelled, _) = watch::channel(false); + let catch_up_wake = Arc::new(tokio::sync::Notify::new()); + let flush_primary = primary.clone(); + let checkpoint_state = primary.as_ref().map(|primary| { + Arc::new(CheckpointWorker::new( + primary.clone(), + store.clone(), + provider.clone(), + )) + }); + let checkpoint_reader = Arc::new(CheckpointReader::new(store.clone())); + if let Some(worker) = checkpoint_state { + let cancellation = catch_up_cancelled.subscribe(); + let mut activated = activation.subscribe(); + drop(tokio::spawn(async move { + if wait_for_start(&mut activated).await { + worker.run(checkpoint_interval, cancellation).await; + } + })); + } + let catch_up_workers = primary.map_or_else(Vec::new, |primary| { + targets + .into_iter() + .map(|target| { + Arc::new( + CatchUpWorker::new( + target.backend_id, + primary.clone(), + target.backend, + store.clone(), + provider.clone(), + ) + .with_checkpoint_consumer(checkpoint_reader.clone()) + .with_cancellation(catch_up_cancelled.subscribe()), + ) + }) + .collect::>() + }); + let worker = FlushWorker::new( + store.clone(), + provider, + initial_partitions, + backup_names, + flush_primary, + ); + let mut activated = activation.subscribe(); + drop(tokio::spawn(async move { + if wait_for_start(&mut activated).await { + worker.recover_directory_events().await; + if *activated.borrow() == Some(true) { + worker.run(receiver, worker_pending_events).await; + } + } + worker_running.store(false, Ordering::SeqCst); + })); + for worker in catch_up_workers { + let mut cancelled = catch_up_cancelled.subscribe(); + let wake = catch_up_wake.clone(); + let mut activated = activation.subscribe(); + drop(tokio::spawn(async move { + if !wait_for_start(&mut activated).await { + return; + } + while !*cancelled.borrow() { + if let Err(error) = worker.run_once().await { + debug!(error = %error, "multi-write catch-up round failed"); + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(300)) => {} + _ = wake.notified() => {} + changed = cancelled.changed() => { + if changed.is_err() || *cancelled.borrow() { + break; + } + } + } + } + })); + } + Self { + sender: StdMutex::new(Some(sender)), + activation, + import_worker: StdMutex::new(None), + store, + catch_up_cancelled, + catch_up_wake, + running, + pending_events, + } + } + + /// Attach the full-data import task to this runtime's lifecycle. + pub(crate) fn set_import_worker( + &self, + worker: impl std::future::Future> + Send + 'static, + ) { + let mut activated = self.activation.subscribe(); + let worker = tokio::spawn(async move { + if wait_for_start(&mut activated).await { + worker.await + } else { + Ok(()) + } + }); + *self + .import_worker + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some(worker); + } + + /// Release prepared runtime work after the mount becomes visible. + pub(crate) fn activate(&self) { + self.activation.send_replace(Some(true)); + } + + /// Submit one event without waiting for metadata work or taking business locks. + pub fn submit(&self, event: PendingEvent) { + let failed = { + let sender = self + .sender + .lock() + .unwrap_or_else(|error| error.into_inner()); + match sender.as_ref() { + Some(sender) => { + self.pending_events.fetch_add(1, Ordering::Relaxed); + sender.send(event).err().map(|error| { + self.pending_events.fetch_sub(1, Ordering::Relaxed); + error.0 + }) + } + None => Some(event), + } + }; + if let Some(event) = failed { + self.store.record_worker_error(MultiWriteWorker::Flush); + error!( + operation_id = %event.operation_id, + account = %event.account_id, + event_kind = ?event.kind, + "multi-write event queue is closed" + ); + } + self.catch_up_wake.notify_waiters(); + } + + /// Return events still waiting in the foreground channel. + pub(crate) fn pending_event_count(&self) -> usize { + self.pending_events.load(Ordering::Relaxed) + } + + /// Return whether the flush worker has not exited yet. + pub fn is_running(&self) -> bool { + self.running.load(Ordering::SeqCst) + } +} + +impl Drop for MultiWriteRuntime { + /// Signal background tasks when the owning wrapper is dropped. + fn drop(&mut self) { + self.sender + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + self.activation.send_replace(Some(false)); + if let Some(worker) = self + .import_worker + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + worker.abort(); + } + self.catch_up_cancelled.send_replace(true); + self.catch_up_wake.notify_waiters(); + } +} + +/// Wait until a prepared runtime gate is started or stopped. +async fn wait_for_start(gate: &mut watch::Receiver>) -> bool { + gate.wait_for(|state| state.is_some()) + .await + .is_ok_and(|state| *state == Some(true)) +} + +#[cfg(test)] +#[path = "tests/test_runtime.rs"] +mod tests; + +/// Drains queued events and persists normal file events by account and scope. +pub struct FlushWorker { + store: Arc, + provider: Arc, + primary: Option>, + router: AccountRouter, + initial_partitions: u32, + backup_names: Vec, +} + +impl FlushWorker { + /// Create a worker using one metadata store and persistence provider. + pub fn new( + store: Arc, + provider: Arc, + initial_partitions: u32, + backup_names: Vec, + primary: Option>, + ) -> Self { + Self { + router: AccountRouter::new(store.clone()), + store, + provider, + primary, + initial_partitions, + backup_names, + } + } + + /// Drain the receiver until every sender is closed and queued event is consumed. + pub async fn run( + mut self, + mut receiver: mpsc::UnboundedReceiver, + pending_events: Arc, + ) { + while let Some(first) = receiver.recv().await { + let mut batch = vec![first]; + while let Ok(event) = receiver.try_recv() { + batch.push(event); + } + let attempted = batch.len(); + self.flush_batch(batch).await; + pending_events.fetch_sub(attempted, Ordering::Relaxed); + } + } + + /// Flush one batch while recording and dropping failed events. + async fn flush_batch(&mut self, batch: Vec) { + let mut failed = false; + let mut accounts = HashMap::>::new(); + for event in batch { + if matches!(event.kind, PendingEventKind::DeleteAccount) { + accounts + .entry(SYSTEM_DIR.to_string()) + .or_default() + .push(Self::delete_scope_event(event)); + continue; + } + accounts + .entry(event.account_id.clone()) + .or_default() + .push(event); + } + + for (account, events) in accounts { + match self.account_exists(&account).await { + Ok(false) => continue, + Err(error) => { + failed = true; + error!(account = %account, error = %error, + "multi-write primary account check failed"); + continue; + } + Ok(true) => {} + } + match self + .store + .initialize_account(&account, self.initial_partitions, &self.backup_names) + .await + { + Ok(partitions) => { + if let Err(error) = bootstrap_account( + self.provider.as_ref(), + self.store.as_ref(), + &account, + &partitions, + ) + .await + { + failed = true; + error!(account = %account, error = %error, + "multi-write provider bootstrap failed"); + continue; + } + } + Err(error) => { + failed = true; + error!( + account = %account, + error = %error, + "multi-write account initialization failed" + ); + continue; + } + } + + let mut groups = HashMap::)>::new(); + for mut event in events { + match event.kind { + PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => { + match self.router.route(&account, &event.path).await { + Ok(route) => { + event.route_hint = Some(route.clone()); + groups + .entry(route.scope.clone()) + .or_insert_with(|| (route, Vec::new())) + .1 + .push(event); + } + Err(error) => { + failed = true; + error!( + operation_id = %event.operation_id, + account = %account, + event_kind = ?event.kind, + error = %error, + "multi-write event routing failed" + ); + } + } + } + PendingEventKind::Data( + SegmentEventType::RemoveTree + | SegmentEventType::MoveTree + | SegmentEventType::DeleteScope, + ) => { + if let Err(error) = self.flush_directory_event(&account, &event).await { + failed = true; + error!( + operation_id = %event.operation_id, + account = %account, + event_kind = ?event.kind, + error = %error, + "multi-write directory event persistence failed" + ); + } + } + PendingEventKind::DeleteAccount => unreachable!(), + } + } + + for (scope, (route, events)) in groups { + if let Err(error) = self.provider.flush(Some(&route), events).await { + failed = true; + error!( + account = %account, + partition = scope.partition_id, + epoch = scope.epoch, + error = %error, + "multi-write metadata flush failed" + ); + } + } + } + if failed { + self.store.record_worker_error(MultiWriteWorker::Flush); + } + } + + /// Convert one account-root deletion into a durable system-scope marker. + fn delete_scope_event(mut event: PendingEvent) -> PendingEvent { + event.account_id = SYSTEM_DIR.to_string(); + event.kind = PendingEventKind::Data(SegmentEventType::DeleteScope); + event.route_hint = None; + event + } + + /// Check account membership through the user registry when the primary view is available. + async fn account_exists(&self, account_id: &str) -> Result { + if account_id == SYSTEM_DIR { + return Ok(true); + } + let Some(primary) = &self.primary else { + return self.store.account_exists(account_id).await; + }; + Ok(read_active_accounts(primary.as_ref()) + .await? + .contains(account_id)) + } + + /// Register and persist one directory marker to every captured stable partition. + async fn flush_directory_event(&self, account: &str, event: &PendingEvent) -> Result<()> { + let scopes = self.register_directory_event(account, event).await?; + let mut failed = false; + for scope in scopes { + let result = match self.find_or_append_marker(&scope, event).await { + Ok(position) => { + self.backfill_position(account, event.operation_id, position) + .await + } + Err(error) => Err(error), + }; + if let Err(error) = result { + failed = true; + error!( + operation_id = %event.operation_id, account = %account, + partition = scope.partition_id, epoch = scope.epoch, error = %error, + "multi-write directory marker scope remains incomplete" + ); + } + } + if failed { + Err(Error::internal("directory marker scopes remain incomplete")) + } else { + Ok(()) + } + } + + /// Register one operation idempotently and return its missing stable scopes. + async fn register_directory_event( + &self, + account: &str, + event: &PendingEvent, + ) -> Result> { + let persisted = Self::directory_event(event)?; + self.store + .update_partitions_manifest(account, move |manifest| { + let index = manifest + .directory_events + .iter() + .position(|candidate| candidate.op_id == persisted.op_id); + if let Some(index) = index { + let existing = &manifest.directory_events[index]; + if existing.operation != persisted.operation + || existing.source_path != persisted.source_path + || existing.destination_path != persisted.destination_path + { + return Err(Error::invalid_operation( + "directory op_id was reused with different content", + )); + } + } else { + manifest.directory_events.push(persisted); + } + let event = manifest + .directory_events + .iter() + .find(|candidate| candidate.op_id == event.operation_id) + .unwrap(); + Ok(Self::missing_scopes(account, manifest, &event.positions)) + }) + .await + } + + /// Return Stable scopes that do not yet have a marker position. + fn missing_scopes( + account: &str, + manifest: &PartitionsManifest, + positions: &[MarkerPosition], + ) -> Vec { + let completed = positions + .iter() + .map(|position| position.partition_id) + .collect::>(); + manifest + .partitions + .iter() + .filter(|(id, partition)| { + partition.state == PartitionState::Stable && !completed.contains(id) + }) + .map(|(&partition_id, _)| ScopeKey { + account_id: account.to_string(), + partition_id, + epoch: manifest.epoch, + }) + .collect() + } + + /// Convert one queued directory marker into its persisted notification. + fn directory_event(event: &PendingEvent) -> Result { + let operation = match event.kind { + PendingEventKind::Data(SegmentEventType::RemoveTree) => DirectoryOperation::RemoveTree, + PendingEventKind::Data(SegmentEventType::MoveTree) => DirectoryOperation::MoveTree, + PendingEventKind::Data(SegmentEventType::DeleteScope) => { + DirectoryOperation::DeleteScope + } + _ => { + return Err(Error::invalid_operation( + "directory notification requires a tree event", + )) + } + }; + Ok(DirectoryEvent { + op_id: event.operation_id, + operation, + source_path: event.path.clone(), + destination_path: event.destination_path.clone(), + positions: Vec::new(), + }) + } + + /// Idempotently persist one marker position under the manifest Exact lock. + async fn backfill_position( + &self, + account: &str, + operation_id: uuid::Uuid, + position: MarkerPosition, + ) -> Result<()> { + self.store + .update_partitions_manifest(account, move |manifest| { + let event = manifest + .directory_events + .iter_mut() + .find(|event| event.op_id == operation_id) + .ok_or_else(|| Error::not_found(operation_id.to_string()))?; + if let Some(existing) = event + .positions + .iter() + .find(|existing| existing.partition_id == position.partition_id) + { + if existing != &position { + return Err(Error::invalid_operation( + "conflicting directory marker positions", + )); + } + return Ok(()); + } + event.positions.push(position); + event + .positions + .sort_by_key(|position| position.partition_id); + Ok(()) + }) + .await + } + + /// Recover every incomplete notification discoverable before queue processing starts. + async fn recover_directory_events(&self) { + let accounts = match self.store.initialized_accounts().await { + Ok(accounts) => accounts, + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Flush); + error!(error = %error, "multi-write directory recovery discovery failed"); + return; + } + }; + for account in accounts { + match self.recover_account(&account).await { + Ok(()) | Err(Error::NotFound(_)) => {} + Err(error) => { + self.store.record_worker_error(MultiWriteWorker::Flush); + error!(account = %account, error = %error, + "multi-write directory recovery failed"); + } + } + } + } + + /// Recover all currently missing marker positions for one account. + async fn recover_account(&self, account: &str) -> Result<()> { + let manifest_path = self.store.paths().account_manifest(account)?.0; + let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?; + manifest.validate()?; + let mut failed = false; + for persisted in &manifest.directory_events { + let event = PendingEvent { + operation_id: persisted.op_id, + account_id: account.to_string(), + kind: PendingEventKind::Data(match persisted.operation { + DirectoryOperation::RemoveTree => SegmentEventType::RemoveTree, + DirectoryOperation::MoveTree => SegmentEventType::MoveTree, + DirectoryOperation::DeleteScope => SegmentEventType::DeleteScope, + }), + path: persisted.source_path.clone(), + destination_path: persisted.destination_path.clone(), + route_hint: None, + }; + for scope in Self::missing_scopes(account, &manifest, &persisted.positions) { + let result = self.find_or_append_marker(&scope, &event).await; + match result { + Ok(position) => { + if let Err(error) = self + .backfill_position(account, event.operation_id, position) + .await + { + failed = true; + error!(operation_id = %event.operation_id, account = %account, + partition = scope.partition_id, error = %error, + "multi-write recovered marker position update failed"); + } + } + Err(error) => { + failed = true; + error!(operation_id = %event.operation_id, account = %account, + partition = scope.partition_id, error = %error, + "multi-write directory marker recovery scope failed"); + } + } + } + } + if failed { + Err(Error::internal( + "directory marker recovery remains incomplete", + )) + } else { + Ok(()) + } + } + + /// Reuse a retained sealed marker or append it when none exists. + async fn find_or_append_marker( + &self, + scope: &ScopeKey, + event: &PendingEvent, + ) -> Result { + match self.find_marker(scope, event).await? { + Some(position) => Ok(position), + None => self.provider.append_marker(scope, event).await, + } + } + + /// Find one operation marker in currently retained sealed segments. + async fn find_marker( + &self, + scope: &ScopeKey, + event: &PendingEvent, + ) -> Result> { + let manifest = self.provider.read_manifest(scope).await?; + let directory = self + .store + .paths() + .segments_dir(&scope.account_id, scope.partition_id)? + .0; + for descriptor in manifest + .segments + .iter() + .filter(|segment| segment.segment_to_seq.is_some()) + { + let bytes = self + .store + .read_bytes(&format!("{directory}/{}", descriptor.path)) + .await?; + if bytes.len() as u64 != descriptor.byte_len + || descriptor.checksum.as_deref() != Some(&sha256_hex(&bytes)) + { + return Err(Error::Serialization( + "sealed marker segment mismatch".to_string(), + )); + } + let decoded = decode_segment(&bytes)?; + if decoded.truncated_tail || decoded.valid_bytes != descriptor.byte_len { + return Err(Error::Serialization( + "sealed marker segment is truncated".to_string(), + )); + } + if let Some(record) = decoded + .records + .iter() + .find(|record| record.op_id == Some(event.operation_id)) + { + let PendingEventKind::Data(kind) = event.kind else { + unreachable!() + }; + if record.event_type != kind + || record.path != event.path + || record.destination_path != event.destination_path + { + return Err(Error::Serialization( + "directory op_id marker mismatch".to_string(), + )); + } + return Ok(Some(MarkerPosition { + partition_id: scope.partition_id, + epoch: scope.epoch, + seq: record.seq, + })); + } + } + Ok(None) + } +} diff --git a/crates/ragfs/src/multibackend/tests/test_catch_up.rs b/crates/ragfs/src/multibackend/tests/test_catch_up.rs new file mode 100644 index 0000000000..2ee8a5a338 --- /dev/null +++ b/crates/ragfs/src/multibackend/tests/test_catch_up.rs @@ -0,0 +1,89 @@ +use super::*; + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::core::context::{FsContextInner, FS_CTX}; +use crate::core::encryption_wrapper::EncryptionWrappedFS; +use crate::core::{MountableFS, PluginConfig}; +use crate::crypto; +use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager}; +use crate::plugins::memfs::MemFileSystem; +use crate::plugins::MemFSPlugin; + +/// Build one encrypted MemFS backup and expose its raw mounted storage. +async fn encrypted_backup() -> (Arc, EncryptionWrappedFS) { + let raw = Arc::new(MountableFS::new()); + raw.register_plugin(MemFSPlugin).await; + raw.mount(PluginConfig::single_backend( + "memfs", + "/mem", + HashMap::new(), + )) + .await + .unwrap(); + let manager = Arc::new(PathLockManager::new( + raw.clone() as Arc, + Arc::new(MemoryPathLockProvider::new()), + PathLockConfig::default(), + )); + raw.set_pathlock_manager(manager.clone()).await; + let encrypted = EncryptionWrappedFS::new( + raw.clone(), + [7_u8; 32], + crypto::PROVIDER_LOCAL, + manager, + "/mem".to_string(), + ); + (raw, encrypted) +} + +/// Verify encrypted writes preserve their historical physical byte count. +#[tokio::test] +async fn encrypted_write_returns_physical_length() { + let (raw, encrypted) = encrypted_backup().await; + let data = b"plain"; + let written = FS_CTX + .scope( + Arc::new(FsContextInner::new("acct")), + encrypted.write("/mem/file.txt", data, 0, WriteFlag::Create), + ) + .await + .unwrap(); + let stored = raw.read("/mem/file.txt", 0, 0).await.unwrap(); + + assert_eq!(written, stored.len() as u64); + assert!(written > data.len() as u64); +} + +/// Verify catch-up accepts an encrypted backend's physical byte count. +#[tokio::test] +async fn reconcile_file_accepts_encrypted_write_length() { + let primary = MemFileSystem::new(); + primary + .ensure_parent_dirs("/mem/file.txt", 0o755) + .await + .unwrap(); + primary + .write("/mem/file.txt", b"plain", 0, WriteFlag::Create) + .await + .unwrap(); + let (_, encrypted) = encrypted_backup().await; + + FS_CTX + .scope( + Arc::new(FsContextInner::new("acct")), + reconcile_file(&primary, &encrypted, "/mem/file.txt"), + ) + .await + .unwrap(); + + let copied = FS_CTX + .scope( + Arc::new(FsContextInner::new("acct")), + encrypted.read("/mem/file.txt", 0, 0), + ) + .await + .unwrap(); + assert_eq!(copied, b"plain"); +} diff --git a/crates/ragfs/src/multibackend/tests/test_codec.rs b/crates/ragfs/src/multibackend/tests/test_codec.rs new file mode 100644 index 0000000000..2dc542feab --- /dev/null +++ b/crates/ragfs/src/multibackend/tests/test_codec.rs @@ -0,0 +1,35 @@ +use super::*; +use uuid::Uuid; + +/// Verifies segment round trips and recovery from an incomplete final frame. +#[test] +fn segment_round_trip_recovers_complete_prefix_from_truncated_tail() { + let records = vec![ + SegmentRecord { + seq: 1, + event_type: SegmentEventType::Write, + path: "/local/account/file.txt".to_string(), + op_id: None, + destination_path: None, + }, + SegmentRecord { + seq: 2, + event_type: SegmentEventType::MoveTree, + path: "/local/account/source".to_string(), + op_id: Some(Uuid::from_u128(7)), + destination_path: Some("/local/account/destination".to_string()), + }, + ]; + + let encoded = encode_segment(&records).unwrap(); + let decoded = decode_segment(&encoded).unwrap(); + assert_eq!(decoded.records, records); + assert_eq!(decoded.valid_bytes, encoded.len() as u64); + assert!(!decoded.truncated_tail); + + let first_record_bytes = encode_segment(&records[..1]).unwrap(); + let truncated = decode_segment(&encoded[..encoded.len() - 1]).unwrap(); + assert_eq!(truncated.records, records[..1]); + assert_eq!(truncated.valid_bytes, first_record_bytes.len() as u64); + assert!(truncated.truncated_tail); +} diff --git a/crates/ragfs/src/multibackend/tests/test_model.rs b/crates/ragfs/src/multibackend/tests/test_model.rs new file mode 100644 index 0000000000..1f66781109 --- /dev/null +++ b/crates/ragfs/src/multibackend/tests/test_model.rs @@ -0,0 +1,45 @@ +use super::*; +use std::collections::BTreeMap; +use uuid::Uuid; + +/// Verifies complete routing and directory marker ownership invariants. +#[test] +fn partitions_manifest_rejects_route_gaps_and_stale_markers() { + let manifest = PartitionsManifest { + version: 1, + epoch: 3, + vbuckets: VBUCKETS, + partitions: BTreeMap::from([( + 0, + PartitionEntry { + state: PartitionState::Stable, + }, + )]), + routes: vec![RouteEntry { + start: 0, + end: VBUCKETS - 1, + partition: 0, + }], + directory_events: vec![DirectoryEvent { + op_id: Uuid::from_u128(1), + operation: DirectoryOperation::RemoveTree, + source_path: "/local/account/removed".to_string(), + destination_path: None, + positions: vec![MarkerPosition { + partition_id: 0, + epoch: 3, + seq: 1, + }], + }], + }; + + assert!(manifest.validate().is_ok()); + + let mut route_gap = manifest.clone(); + route_gap.routes[0].start = 1; + assert!(route_gap.validate().is_err()); + + let mut stale_marker = manifest; + stale_marker.directory_events[0].positions[0].epoch = 2; + assert!(stale_marker.validate().is_err()); +} diff --git a/crates/ragfs/src/multibackend/tests/test_runtime.rs b/crates/ragfs/src/multibackend/tests/test_runtime.rs new file mode 100644 index 0000000000..e9523af265 --- /dev/null +++ b/crates/ragfs/src/multibackend/tests/test_runtime.rs @@ -0,0 +1,97 @@ +use super::*; +use crate::core::{FileSystem, WriteFlag}; +use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager}; +use crate::multibackend::provider::FilesystemProvider; +use crate::plugins::memfs::MemFileSystem; + +/// Verifies deleted registry entries are excluded while null deletions stay active. +#[tokio::test] +async fn active_accounts_exclude_completed_deletions() { + let primary = MemFileSystem::new(); + let registry = serde_json::json!({ + "accounts": { + "active": {}, + "pending": {"deletion": null}, + "deleted": {"deletion": {"deleted_at_ns": 1}} + } + }); + primary + .ensure_parent_dirs("/_system/accounts.json", 0o755) + .await + .unwrap(); + primary + .write( + "/_system/accounts.json", + &serde_json::to_vec(®istry).unwrap(), + 0, + WriteFlag::Create, + ) + .await + .unwrap(); + + let accounts = read_active_accounts(&primary).await.unwrap(); + assert_eq!( + accounts, + HashSet::from(["active".to_string(), "pending".to_string()]) + ); +} + +/// Verifies delete events are persisted in the system scope and remove backup data. +#[tokio::test] +async fn delete_account_event_removes_backup_scope() { + let primary: Arc = Arc::new(MemFileSystem::new()); + let backup: Arc = Arc::new(MemFileSystem::new()); + primary + .ensure_parent_dirs("/_system/accounts.json", 0o755) + .await + .unwrap(); + primary + .write( + "/_system/accounts.json", + br#"{"accounts":{"account":{}}}"#, + 0, + WriteFlag::Create, + ) + .await + .unwrap(); + backup.mkdir("/account", 0o755).await.unwrap(); + + let manager = Arc::new(PathLockManager::new( + primary.clone(), + Arc::new(MemoryPathLockProvider::new()), + PathLockConfig::default(), + )); + let store = Arc::new(MetadataStore::new(primary.clone(), manager, "/local").unwrap()); + let provider = Arc::new(FilesystemProvider::new(store.clone())); + let catch_up = Arc::new(CatchUpWorker::new( + "backup".to_string(), + primary.clone(), + backup.clone(), + store.clone(), + provider.clone(), + )); + let mut worker = FlushWorker::new( + store, + provider, + 1, + vec!["backup".to_string()], + Some(primary), + ); + + worker + .flush_batch(vec![PendingEvent { + operation_id: uuid::Uuid::from_u128(1), + account_id: "account".to_string(), + kind: PendingEventKind::DeleteAccount, + path: "/local/account".to_string(), + destination_path: None, + route_hint: None, + }]) + .await; + catch_up.run_once().await.unwrap(); + + assert!(matches!( + backup.stat("/account").await.unwrap_err(), + Error::NotFound(_) + )); +} diff --git a/crates/ragfs/src/multibackend/types.rs b/crates/ragfs/src/multibackend/types.rs index 78bb71d664..09a4f53ec5 100644 --- a/crates/ragfs/src/multibackend/types.rs +++ b/crates/ragfs/src/multibackend/types.rs @@ -2,6 +2,8 @@ use std::sync::Arc; +#[cfg(feature = "cache")] +use crate::cache_runtime::CacheRuntime; use crate::lock::PathLockManager; /// Encryption context used while assembling one multi-backend mount. @@ -15,6 +17,9 @@ pub struct MultiBackendBuildContext { pub pathlock_manager: Arc, /// Mount prefix used to restore manager-visible backend paths. pub backend_prefix: String, + /// Optional shared cache runtime used by cache-backed metadata. + #[cfg(feature = "cache")] + pub cache_runtime: Option>, } impl MultiBackendBuildContext { diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs index 67b50a7dcc..463f7ae3b4 100644 --- a/crates/ragfs/src/plugins/localfs/mod.rs +++ b/crates/ragfs/src/plugins/localfs/mod.rs @@ -897,6 +897,8 @@ impl FileSystem for LocalFileSystem { let mut file = options.open(&local_path).map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { Error::NotFound(path.to_string()) + } else if e.kind() == std::io::ErrorKind::AlreadyExists { + Error::AlreadyExists(path.to_string()) } else { Error::plugin(format!("failed to open file: {}", e)) } diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rs deleted file mode 100644 index 4c31ab2887..0000000000 --- a/crates/ragfs/tests/cache_wrapper.rs +++ /dev/null @@ -1,1478 +0,0 @@ -use async_trait::async_trait; -use ragfs::cache::{ - CacheDecision, CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem, -}; -use ragfs::cache_runtime::{CacheRuntime, MemoryMockProvider}; -use ragfs::core::{ - FsContextInner, GrepOptions, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX, -}; -use ragfs::plugins::MemFileSystem; -use ragfs::{Error, FileInfo, FileSystem, Result, WriteFlag}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -#[derive(Clone)] -struct CountingFileSystem { - inner: Arc, - reads: Arc, - read_dirs: Arc, - stats: Arc, - greps: Arc, - trees: Arc, - read_delay: Duration, - partial_remove_path: Option, -} - -impl CountingFileSystem { - fn new() -> Self { - Self { - inner: Arc::new(MemFileSystem::new()), - reads: Arc::new(AtomicU64::new(0)), - read_dirs: Arc::new(AtomicU64::new(0)), - stats: Arc::new(AtomicU64::new(0)), - greps: Arc::new(AtomicU64::new(0)), - trees: Arc::new(AtomicU64::new(0)), - read_delay: Duration::ZERO, - partial_remove_path: None, - } - } - - fn with_read_delay(mut self, delay: Duration) -> Self { - self.read_delay = delay; - self - } - - fn with_partial_remove_failure(mut self, path: &str) -> Self { - self.partial_remove_path = Some(path.to_string()); - self - } - - fn read_count(&self) -> u64 { - self.reads.load(Ordering::Relaxed) - } - - fn read_dir_count(&self) -> u64 { - self.read_dirs.load(Ordering::Relaxed) - } - - fn stat_count(&self) -> u64 { - self.stats.load(Ordering::Relaxed) - } - - fn grep_count(&self) -> u64 { - self.greps.load(Ordering::Relaxed) - } - - fn tree_count(&self) -> u64 { - self.trees.load(Ordering::Relaxed) - } -} - -#[async_trait] -impl FileSystem for CountingFileSystem { - async fn create(&self, path: &str) -> Result<()> { - self.inner.create(path).await - } - - async fn mkdir(&self, path: &str, mode: u32) -> Result<()> { - self.inner.mkdir(path, mode).await - } - - async fn remove(&self, path: &str) -> Result<()> { - self.inner.remove(path).await - } - - async fn remove_all(&self, path: &str) -> Result<()> { - if let Some(partial_remove_path) = &self.partial_remove_path { - self.inner.remove(partial_remove_path).await?; - return Err(Error::internal(format!( - "remove_all partially removed {partial_remove_path} under {path}" - ))); - } - self.inner.remove_all(path).await - } - - async fn read(&self, path: &str, offset: u64, size: u64) -> Result> { - self.reads.fetch_add(1, Ordering::Relaxed); - if !self.read_delay.is_zero() { - tokio::time::sleep(self.read_delay).await; - } - self.inner.read(path, offset, size).await - } - - async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result { - self.inner.write(path, data, offset, flags).await - } - - async fn read_dir(&self, path: &str) -> Result> { - self.read_dirs.fetch_add(1, Ordering::Relaxed); - self.inner.read_dir(path).await - } - - async fn stat(&self, path: &str) -> Result { - self.stats.fetch_add(1, Ordering::Relaxed); - self.inner.stat(path).await - } - - async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> { - self.inner.rename(old_path, new_path).await - } - - async fn chmod(&self, path: &str, mode: u32) -> Result<()> { - self.inner.chmod(path, mode).await - } - - async fn truncate(&self, path: &str, size: u64) -> Result<()> { - self.inner.truncate(path, size).await - } - - async fn grep( - &self, - path: &str, - pattern: &str, - options: GrepOptions<'_>, - ) -> Result { - self.greps.fetch_add(1, Ordering::Relaxed); - self.inner.grep(path, pattern, options).await - } - - async fn tree_directory( - &self, - path: &str, - show_hidden: bool, - node_limit: Option, - level_limit: Option, - ) -> Result> { - self.trees.fetch_add(1, Ordering::Relaxed); - self.inner - .tree_directory(path, show_hidden, node_limit, level_limit) - .await - } -} - -fn cached_fs(backend: CountingFileSystem) -> (Arc, Arc) { - cached_fs_with_policy(backend, CachePolicy::default()) -} - -fn cached_fs_with_policy( - backend: CountingFileSystem, - policy: CachePolicy, -) -> (Arc, Arc) { - let provider = Arc::new(MemoryMockProvider::new()); - let fs = Arc::new(CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider.clone()), - CacheNamespace::new("test"), - policy, - )); - (fs, provider) -} - -fn cached_fs_with_tracking_provider( - backend: CountingFileSystem, - policy: CachePolicy, -) -> (Arc, Arc) { - cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(MemoryMockProvider::new())) -} - -fn cached_fs_with_tracking_provider_instance( - backend: CountingFileSystem, - policy: CachePolicy, - provider: Arc, -) -> (Arc, Arc) { - let fs = Arc::new(CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider.clone()), - CacheNamespace::new("tracking"), - policy, - )); - (fs, provider) -} - -#[test] -fn cache_policy_traversal_mode_defaults_to_backend() { - assert_eq!( - CachePolicy::default().traversal_mode(), - CacheTraversalMode::Backend - ); -} - -#[tokio::test] -async fn cached_filesystem_accepts_the_unified_runtime_without_changing_read_through() { - let backend = CountingFileSystem::new(); - backend - .write("/runtime.md", b"runtime", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let fs = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory(), - CacheNamespace::new("runtime"), - CachePolicy::default(), - ); - - assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime"); - assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime"); - assert_eq!(probe.read_count(), 1); -} - -#[tokio::test] -async fn unified_runtime_failure_still_falls_back_to_backend_reads() { - let backend = CountingFileSystem::new(); - backend - .write("/fallback.txt", b"backend", 0, WriteFlag::Create) - .await - .unwrap(); - let runtime = CacheRuntime::memory(); - runtime.close().await.unwrap(); - let cached = CachedFileSystem::with_runtime( - Box::new(backend), - runtime, - CacheNamespace::new("runtime-fail-open"), - CachePolicy::default(), - ); - - assert_eq!( - cached.read("/fallback.txt", 0, 0).await.unwrap(), - b"backend" - ); -} - -#[tokio::test] -async fn default_tree_directory_delegates_to_backend() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend - .write("/docs/one.md", b"one", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - let entries = fs.tree_directory("/docs", false, None, None).await.unwrap(); - - assert_eq!(entries.len(), 1); - assert_eq!(probe.tree_count(), 1); - assert_eq!(probe.read_dir_count(), 0); -} - -#[tokio::test] -async fn default_grep_delegates_to_backend() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend - .write("/docs/one.md", b"needle", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - let result = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - - assert_eq!(result.count, 1); - assert_eq!(probe.grep_count(), 1); - assert_eq!(probe.read_dir_count(), 0); - assert_eq!(probe.read_count(), 0); -} - -#[tokio::test] -async fn cached_tree_traversal_reuses_directory_cache_after_warmup() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend - .write("/docs/one.md", b"one", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/two.md", b"two", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let first = fs.tree_directory("/docs", false, None, None).await.unwrap(); - assert_eq!(first.len(), 3); - assert_eq!(probe.tree_count(), 0); - assert_eq!(probe.read_dir_count(), 2); - - let second = fs.tree_directory("/docs", false, None, None).await.unwrap(); - assert_eq!(second.len(), 3); - assert_eq!( - probe.read_dir_count(), - 2, - "second tree should reuse cached read_dir entries" - ); -} - -#[tokio::test] -async fn cached_grep_traversal_reuses_directory_and_file_cache_after_warmup() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let first = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - assert_eq!(first.count, 2); - assert_eq!(probe.grep_count(), 0); - assert_eq!(probe.read_dir_count(), 2); - assert_eq!(probe.read_count(), 2); - assert_eq!(probe.stat_count(), 1); - - let second = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - assert_eq!(second.count, 2); - assert_eq!( - probe.read_dir_count(), - 2, - "second grep should reuse cached read_dir entries" - ); - assert_eq!( - probe.read_count(), - 2, - "second grep should reuse cached file contents" - ); - assert_eq!( - probe.stat_count(), - 2, - "second grep should only stat the query root, not every cached entry" - ); -} - -#[tokio::test] -async fn cached_grep_includes_context_from_cached_content() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend - .write( - "/docs/a.md", - b"first\nbefore\nneedle\nafter", - 0, - WriteFlag::Create, - ) - .await - .unwrap(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let result = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - before_context: 2, - after_context: 1, - ..Default::default() - }, - ) - .await - .unwrap(); - - let matched = &result.matches[0]; - assert_eq!(matched.before_context.as_ref().unwrap()[0].content, "first"); - assert_eq!( - matched.before_context.as_ref().unwrap()[1].content, - "before" - ); - assert_eq!(matched.after_context.as_ref().unwrap()[0].content, "after"); -} - -#[tokio::test] -async fn cached_grep_batches_generation_validation_after_warmup() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create) - .await - .unwrap(); - let (fs, provider) = cached_fs_with_tracking_provider( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - fs.grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - provider.reset_observed_reads(); - - let result = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - - assert_eq!(result.count, 2); - assert!( - provider.batch_get_count() > 0, - "warm cached grep should batch generation validation reads" - ); -} - -#[tokio::test] -async fn cached_grep_memoizes_generation_keys_within_one_traversal() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create) - .await - .unwrap(); - let (fs, provider) = cached_fs_with_tracking_provider( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - fs.grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - provider.reset_observed_reads(); - - let result = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - - assert_eq!(result.count, 2); - let subtree_keys = provider - .observed_read_keys() - .into_iter() - .filter(|key| key.contains(":subtree:")) - .collect::>(); - let unique = subtree_keys - .iter() - .collect::>(); - assert_eq!( - subtree_keys.len(), - unique.len(), - "one cached grep traversal should not re-read the same generation key" - ); -} - -#[tokio::test] -async fn cached_grep_scans_cached_files_with_bounded_concurrency() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - for index in 0..8 { - backend - .write( - &format!("/docs/{index}.md"), - b"needle\nplain", - 0, - WriteFlag::Create, - ) - .await - .unwrap(); - } - let provider = Arc::new(MemoryMockProvider::new().with_get_delay(Duration::from_millis(30))); - let (fs, provider) = cached_fs_with_tracking_provider_instance( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - provider, - ); - - fs.grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - provider.reset_observed_reads(); - - let result = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap(); - - assert_eq!(result.count, 8); - let max_gets = provider.max_concurrent_gets(); - assert!( - max_gets > 1, - "warm cached grep should scan cached file reads concurrently" - ); - assert!( - max_gets <= 8, - "cached grep file scan concurrency should stay bounded" - ); -} - -#[tokio::test] -async fn cached_grep_traversal_matches_default_grep_semantics() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend.mkdir("/docs/skip", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"Needle\nneedle", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/b.md", b"needle\nmiss", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/skip/c.md", b"needle", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let cached = fs - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - case_insensitive: true, - node_limit: Some(2), - exclude_path: Some("/docs/skip"), - level_limit: Some(1), - ..Default::default() - }, - ) - .await - .unwrap(); - let direct_result = direct - .grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - case_insensitive: true, - node_limit: Some(2), - exclude_path: Some("/docs/skip"), - level_limit: Some(1), - ..Default::default() - }, - ) - .await - .unwrap(); - - let tuples = |result: &GrepResult| { - result - .matches - .iter() - .map(|m| (m.file.clone(), m.line, m.content.clone())) - .collect::>() - }; - assert_eq!(tuples(&cached), tuples(&direct_result)); -} - -#[tokio::test] -async fn cached_grep_traversal_falls_back_for_multiwrite_backend() { - let primary = CountingFileSystem::new(); - primary.mkdir("/docs", 0o755).await.unwrap(); - primary - .write("/docs/a.md", b"needle", 0, WriteFlag::Create) - .await - .unwrap(); - let primary_probe = primary.clone(); - let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary)) - .build() - .unwrap(); - let fs = CachedFileSystem::with_runtime( - Box::new(multiwrite), - CacheRuntime::memory(), - CacheNamespace::new("grep-multiwrite"), - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let ctx = Arc::new(FsContextInner::new("acct")); - let result = FS_CTX - .scope(ctx, async { - fs.grep( - "/docs", - "needle", - GrepOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .unwrap() - }) - .await; - - assert_eq!(result.count, 1); - assert_eq!( - primary_probe.grep_count(), - 1, - "multi-write backends must keep their backend grep path" - ); - assert_eq!( - primary_probe.read_dir_count(), - 0, - "cached traversal must not bypass multi-write grep semantics" - ); -} - -#[tokio::test] -async fn cached_tree_traversal_matches_default_tree_semantics() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend.mkdir("/docs/sub", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"a", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/.hidden.md", b"hidden", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/docs/sub/b.md", b"b", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let cached = fs - .tree_directory("/docs", false, None, Some(1)) - .await - .unwrap(); - let direct_entries = direct - .tree_directory("/docs", false, None, Some(1)) - .await - .unwrap(); - assert_eq!( - cached - .iter() - .map(|entry| entry.rel_path.as_str()) - .collect::>(), - direct_entries - .iter() - .map(|entry| entry.rel_path.as_str()) - .collect::>() - ); - - let with_hidden = fs.tree_directory("/docs", true, None, None).await.unwrap(); - assert!( - with_hidden - .iter() - .any(|entry| entry.rel_path == ".hidden.md"), - "show_hidden=true should include hidden files" - ); -} - -#[tokio::test] -async fn cached_tree_traversal_returns_oversized_directories_without_caching_them() { - let backend = CountingFileSystem::new(); - backend.mkdir("/large", 0o755).await.unwrap(); - for index in 0..4097 { - backend - .write( - &format!("/large/{index:04}.txt"), - b"x", - 0, - WriteFlag::Create, - ) - .await - .unwrap(); - } - let probe = backend.clone(); - let (fs, _) = cached_fs_with_policy( - backend, - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - assert_eq!( - fs.tree_directory("/large", false, None, None) - .await - .unwrap() - .len(), - 4097 - ); - assert_eq!( - fs.tree_directory("/large", false, None, None) - .await - .unwrap() - .len(), - 4097 - ); - assert_eq!( - probe.read_dir_count(), - 2, - "oversized directories should not be cached by tree traversal" - ); -} - -#[tokio::test] -async fn cached_tree_traversal_falls_back_when_provider_is_unavailable() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend - .write("/docs/a.md", b"a", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let provider = Arc::new(MemoryMockProvider::new()); - provider.set_unavailable(true); - let fs = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider), - CacheNamespace::new("tree-unavailable"), - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - assert_eq!( - fs.tree_directory("/docs", false, None, None) - .await - .unwrap() - .len(), - 1 - ); - assert_eq!( - fs.tree_directory("/docs", false, None, None) - .await - .unwrap() - .len(), - 1 - ); - assert_eq!(probe.read_dir_count(), 2); - assert!(fs.metrics().snapshot().errors >= 1); -} - -#[tokio::test] -async fn cached_tree_traversal_falls_back_for_multiwrite_backend() { - let primary = CountingFileSystem::new(); - primary.mkdir("/docs", 0o755).await.unwrap(); - primary - .write("/docs/a.md", b"a", 0, WriteFlag::Create) - .await - .unwrap(); - let primary_probe = primary.clone(); - let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary)) - .build() - .unwrap(); - let fs = CachedFileSystem::with_runtime( - Box::new(multiwrite), - CacheRuntime::memory(), - CacheNamespace::new("tree-multiwrite"), - CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), - ); - - let ctx = Arc::new(FsContextInner::new("acct")); - let entries = FS_CTX - .scope(ctx, async { - fs.tree_directory("/docs", false, None, None).await.unwrap() - }) - .await; - - assert_eq!(entries.len(), 1); - assert_eq!( - primary_probe.tree_count(), - 1, - "multi-write backends must keep their backend tree_directory path" - ); - assert_eq!( - primary_probe.read_dir_count(), - 0, - "cached traversal must not bypass multi-write tree semantics" - ); -} - -#[test] -fn cache_policy_supports_explicit_permission_sensitive_prefixes() { - let policy = CachePolicy::default().with_bypass_prefix("/private"); - - assert!(!policy.cache_file("/private/secret.md", 10)); - assert!(!policy.cache_directory("/private/docs")); - assert!(policy.cache_file("/public/.overview.md", 10)); -} - -#[test] -fn cache_policy_marks_high_value_objects_as_preferred() { - let policy = CachePolicy::default(); - - assert_eq!( - policy.file_decision("/docs/.abstract.md", 128), - CacheDecision::Prefer - ); - assert_eq!( - policy.file_decision("/docs/.overview.md", 128), - CacheDecision::Prefer - ); - assert_eq!( - policy.file_decision("/docs/note.md", 128), - CacheDecision::Cache - ); - assert_eq!(policy.directory_decision("/docs"), CacheDecision::Prefer); -} - -#[tokio::test] -async fn full_file_reads_are_read_through_cached_but_range_reads_bypass() { - let backend = CountingFileSystem::new(); - backend - .write("/note.md", b"hello world", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world"); - assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world"); - assert_eq!(probe.read_count(), 1); - - assert_eq!(fs.read("/note.md", 6, 5).await.unwrap(), b"world"); - assert_eq!(probe.read_count(), 2); - - let metrics = fs.metrics().snapshot(); - assert_eq!(metrics.file_hits, 1); - assert_eq!(metrics.file_misses, 1); - assert_eq!(metrics.backend_fallbacks, 1); -} - -#[tokio::test] -async fn read_dir_is_cached_and_parent_changes_invalidate_it() { - let backend = CountingFileSystem::new(); - backend.mkdir("/docs", 0o755).await.unwrap(); - backend - .write("/docs/one.md", b"one", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1); - assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1); - assert_eq!(probe.read_dir_count(), 1); - - fs.write("/docs/two.md", b"two", 0, WriteFlag::Create) - .await - .unwrap(); - let entries = fs.read_dir("/docs").await.unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(probe.read_dir_count(), 2); - - let metrics = fs.metrics().snapshot(); - assert_eq!(metrics.read_dir_hits, 1); - assert_eq!(metrics.read_dir_misses, 2); - assert!(metrics.invalidations >= 1); -} - -#[tokio::test] -async fn oversized_directories_bypass_directory_cache() { - let cacheable = CountingFileSystem::new(); - cacheable.mkdir("/small", 0o755).await.unwrap(); - for index in 0..4096 { - cacheable - .write( - &format!("/small/{index:04}.txt"), - b"x", - 0, - WriteFlag::Create, - ) - .await - .unwrap(); - } - let cacheable_probe = cacheable.clone(); - let (cacheable_fs, _) = cached_fs(cacheable); - - assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096); - assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096); - assert_eq!(cacheable_probe.read_dir_count(), 1); - - let oversized = CountingFileSystem::new(); - oversized.mkdir("/large", 0o755).await.unwrap(); - for index in 0..4097 { - oversized - .write( - &format!("/large/{index:04}.txt"), - b"x", - 0, - WriteFlag::Create, - ) - .await - .unwrap(); - } - let oversized_probe = oversized.clone(); - let (oversized_fs, _) = cached_fs(oversized); - - assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097); - assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097); - assert_eq!(oversized_probe.read_dir_count(), 2); -} - -#[tokio::test] -async fn all_directory_membership_mutations_invalidate_parent_entries() { - let backend = CountingFileSystem::new(); - backend.mkdir("/root", 0o755).await.unwrap(); - backend.mkdir("/root/old", 0o755).await.unwrap(); - backend - .write("/root/file.txt", b"file", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - - fs.mkdir("/root/new", 0o755).await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 3); - - fs.rename("/root/old", "/root/moved").await.unwrap(); - let names: Vec = fs - .read_dir("/root") - .await - .unwrap() - .into_iter() - .map(|entry| entry.name) - .collect(); - assert!(names.contains(&"moved".to_string())); - assert!(!names.contains(&"old".to_string())); - - fs.remove("/root/file.txt").await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - - fs.remove_all("/root/new").await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(probe.read_dir_count(), 5); -} - -#[tokio::test] -async fn writes_and_deletes_never_leave_stale_file_cache_entries() { - let backend = CountingFileSystem::new(); - backend - .write("/value.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old"); - fs.write("/value.txt", b"new", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new"); - - fs.remove("/value.txt").await.unwrap(); - assert!(fs.read("/value.txt", 0, 0).await.is_err()); - assert_eq!(probe.read_count(), 2); -} - -#[tokio::test] -async fn full_writes_populate_cache_before_returning() { - let backend = CountingFileSystem::new(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - fs.write("/fresh.md", b"fresh", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/fresh.md", 0, 0).await.unwrap(), b"fresh"); - assert_eq!(probe.read_count(), 0); -} - -#[tokio::test] -async fn partial_writes_and_truncate_invalidate_cached_file_contents() { - let backend = CountingFileSystem::new(); - backend - .write("/partial.txt", b"hello", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hello"); - fs.write("/partial.txt", b"X", 1, WriteFlag::None) - .await - .unwrap(); - assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hXllo"); - - fs.truncate("/partial.txt", 2).await.unwrap(); - assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hX"); - assert_eq!(probe.read_count(), 3); -} - -#[tokio::test] -async fn file_rename_invalidates_old_and_new_paths() { - let backend = CountingFileSystem::new(); - backend - .write("/old.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/new.txt", b"historical", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/old.txt", 0, 0).await.unwrap(), b"old"); - assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"historical"); - direct.remove("/new.txt").await.unwrap(); - - fs.rename("/old.txt", "/new.txt").await.unwrap(); - - assert!(fs.read("/old.txt", 0, 0).await.is_err()); - assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"old"); -} - -#[tokio::test] -async fn remove_all_generation_rejects_residual_descendant_cache_entries() { - let backend = CountingFileSystem::new(); - backend.mkdir("/tree", 0o755).await.unwrap(); - backend - .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old"); - fs.remove_all("/tree").await.unwrap(); - - direct.mkdir("/tree", 0o755).await.unwrap(); - direct - .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create) - .await - .unwrap(); - - assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.read_count(), 2); -} - -#[tokio::test] -async fn remove_all_error_invalidates_cached_state_after_partial_mutation() { - let backend = CountingFileSystem::new().with_partial_remove_failure("/tree/deleted.txt"); - backend.mkdir("/tree", 0o755).await.unwrap(); - backend - .write("/tree/deleted.txt", b"deleted", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/tree/survivor.txt", b"survivor", 0, WriteFlag::Create) - .await - .unwrap(); - backend - .write("/unrelated.txt", b"unrelated", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!( - fs.read("/tree/deleted.txt", 0, 0).await.unwrap(), - b"deleted" - ); - assert_eq!( - fs.read("/tree/survivor.txt", 0, 0).await.unwrap(), - b"survivor" - ); - assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated"); - assert_eq!(fs.read_dir("/tree").await.unwrap().len(), 2); - - let error = fs.remove_all("/tree").await.unwrap_err(); - assert!(error.to_string().contains("partially removed")); - - assert!(fs.read("/tree/deleted.txt", 0, 0).await.is_err()); - assert_eq!( - fs.read("/tree/survivor.txt", 0, 0).await.unwrap(), - b"survivor" - ); - assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated"); - let entries = fs.read_dir("/tree").await.unwrap(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].name, "survivor.txt"); - assert_eq!(probe.read_count(), 5); - assert_eq!(probe.read_dir_count(), 2); -} - -#[tokio::test] -async fn shared_provider_generation_bump_invalidates_other_wrappers() { - let backend = CountingFileSystem::new(); - backend.mkdir("/tree", 0o755).await.unwrap(); - backend - .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let probe = backend.clone(); - let runtime = CacheRuntime::memory(); - - let first = CachedFileSystem::with_runtime( - Box::new(backend.clone()), - runtime.clone(), - CacheNamespace::new("shared"), - CachePolicy::default(), - ); - let second = CachedFileSystem::with_runtime( - Box::new(backend), - runtime, - CacheNamespace::new("shared"), - CachePolicy::default(), - ); - - assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old"); - second.remove_all("/tree").await.unwrap(); - - direct.mkdir("/tree", 0o755).await.unwrap(); - direct - .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create) - .await - .unwrap(); - - assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.read_count(), 2); -} - -#[tokio::test] -async fn provider_generation_eviction_after_restart_cannot_revive_old_descendants() { - let backend = CountingFileSystem::new(); - backend.mkdir("/tree", 0o755).await.unwrap(); - backend - .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let probe = backend.clone(); - let provider = Arc::new(MemoryMockProvider::new()); - let first_runtime = CacheRuntime::memory_with_provider(provider.clone()); - - let first = CachedFileSystem::with_runtime( - Box::new(backend.clone()), - first_runtime.clone(), - CacheNamespace::new("restart"), - CachePolicy::default(), - ); - assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old"); - first.remove_all("/tree").await.unwrap(); - - direct.mkdir("/tree", 0o755).await.unwrap(); - direct - .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create) - .await - .unwrap(); - - for key in provider.keys().await { - if key.contains(":subtree:") { - first_runtime.del(&[key]).await.unwrap(); - } - } - drop(first); - drop(first_runtime); - - let restarted = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider), - CacheNamespace::new("restart"), - CachePolicy::default(), - ); - assert_eq!( - restarted.read("/tree/leaf.txt", 0, 0).await.unwrap(), - b"new" - ); - assert_eq!(probe.read_count(), 2); -} - -#[tokio::test] -async fn directory_rename_invalidates_old_and_historical_new_subtrees() { - let backend = CountingFileSystem::new(); - backend.mkdir("/old", 0o755).await.unwrap(); - backend - .write("/old/leaf.txt", b"moved", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/old/leaf.txt", 0, 0).await.unwrap(), b"moved"); - fs.rename("/old", "/new").await.unwrap(); - assert!(fs.read("/old/leaf.txt", 0, 0).await.is_err()); - assert_eq!(fs.read("/new/leaf.txt", 0, 0).await.unwrap(), b"moved"); - - fs.remove_all("/new").await.unwrap(); - direct.mkdir("/new", 0o755).await.unwrap(); - direct - .write("/new/leaf.txt", b"replacement", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!( - fs.read("/new/leaf.txt", 0, 0).await.unwrap(), - b"replacement" - ); -} - -#[tokio::test] -async fn concurrent_misses_share_one_backend_read() { - let backend = CountingFileSystem::new().with_read_delay(Duration::from_millis(30)); - backend - .write("/hot.md", b"hot", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - let mut tasks = Vec::new(); - for _ in 0..12 { - let fs = fs.clone(); - tasks.push(tokio::spawn(async move { - fs.read("/hot.md", 0, 0).await.unwrap() - })); - } - for task in tasks { - assert_eq!(task.await.unwrap(), b"hot"); - } - - assert_eq!(probe.read_count(), 1); - let metrics = fs.metrics().snapshot(); - assert_eq!(metrics.inflight_leaders, 1); - assert_eq!(metrics.inflight_followers, 11); - assert_eq!(metrics.inflight_backend_saved, 11); -} - -#[tokio::test] -async fn cache_policy_bypasses_lock_and_control_files() { - let backend = CountingFileSystem::new(); - backend - .write("/state.lock", b"first", 0, WriteFlag::Create) - .await - .unwrap(); - backend.mkdir("/queue", 0o755).await.unwrap(); - backend - .write("/queue/peek", b"live", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let (fs, _) = cached_fs(backend); - - fs.read("/state.lock", 0, 0).await.unwrap(); - fs.read("/state.lock", 0, 0).await.unwrap(); - fs.read("/queue/peek", 0, 0).await.unwrap(); - fs.read("/queue/peek", 0, 0).await.unwrap(); - - assert_eq!(probe.read_count(), 4); - assert_eq!(fs.metrics().snapshot().policy_bypasses, 4); -} - -#[tokio::test] -async fn failed_invalidation_bypasses_cache_instead_of_serving_stale_data() { - let backend = CountingFileSystem::new(); - backend - .write("/value.txt", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let provider = Arc::new(MemoryMockProvider::new()); - provider.set_delete_failure(true); - let fs = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider), - CacheNamespace::new("delete-failure"), - CachePolicy::default(), - ); - - assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old"); - fs.write("/value.txt", b"new", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.read_count(), 2); - - let metrics = fs.metrics().snapshot(); - assert!(metrics.errors >= 1); - assert!(metrics.policy_bypasses >= 1); -} - -#[tokio::test] -async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() { - let backend = CountingFileSystem::new(); - backend - .write("/available.txt", b"backend", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let provider = Arc::new(MemoryMockProvider::new()); - provider.set_unavailable(true); - let fs = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(provider), - CacheNamespace::new("unavailable"), - CachePolicy::default(), - ); - - assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend"); - assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend"); - assert_eq!(probe.read_count(), 2); - - let metrics = fs.metrics().snapshot(); - assert!(metrics.errors >= 2); - assert!(metrics.policy_bypasses >= 1); -} - -#[tokio::test] -async fn generation_backfill_is_bounded_and_continues_after_one_set_failure() { - let backend = CountingFileSystem::new(); - let mut current = String::new(); - for component in ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l"] { - current.push('/'); - current.push_str(component); - backend.mkdir(¤t, 0o755).await.unwrap(); - } - let path = "/a/b/c/d/e/f/g/h/i/j/k/l/file.md"; - backend - .write(path, b"generation", 0, WriteFlag::Create) - .await - .unwrap(); - - let provider = Arc::new(MemoryMockProvider::new().with_set_delay(Duration::from_millis(20))); - provider.fail_next_set_matching(":subtree:"); - let fs = CachedFileSystem::with_runtime( - Box::new(backend), - CacheRuntime::memory_with_provider(Arc::clone(&provider)), - CacheNamespace::new("generation-set-failure"), - CachePolicy::default(), - ); - - assert_eq!(fs.read(path, 0, 0).await.unwrap(), b"generation"); - - let keys = provider.keys().await; - assert_eq!( - keys.iter().filter(|key| key.contains(":subtree:")).count(), - 13, - "one of the fourteen ancestor generation writes should fail" - ); - assert_eq!( - keys.iter().filter(|key| key.contains(":file:")).count(), - 1, - "a generation write failure must not prevent the file cache fill" - ); - assert_eq!(fs.metrics().snapshot().errors, 1); - assert!( - (2..=8).contains(&provider.max_concurrent_sets()), - "generation writes should run concurrently with a fixed upper bound" - ); -} - -#[tokio::test] -async fn metrics_cover_operations_bytes_latency_and_errors() { - let backend = CountingFileSystem::new(); - backend - .write("/metrics.txt", b"metrics", 0, WriteFlag::Create) - .await - .unwrap(); - let (fs, _) = cached_fs(backend); - - assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics"); - assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics"); - fs.write("/metrics.txt", b"updated", 0, WriteFlag::Truncate) - .await - .unwrap(); - - let metrics = fs.metrics().snapshot(); - assert_eq!(metrics.file_hits, 1); - assert_eq!(metrics.file_misses, 1); - assert_eq!(metrics.backend_fallbacks, 1); - assert!(metrics.puts >= 1); - assert!(metrics.deletes >= 1); - assert!(metrics.invalidations >= 1); - assert_eq!(metrics.backend_bytes, 7); - assert_eq!(metrics.cache_bytes, 7); - assert!(metrics.get_latency_ns > 0); - assert!(metrics.put_latency_ns > 0); - assert!(metrics.delete_latency_ns > 0); - assert_eq!(metrics.errors, 0); -} diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md index 79c406d96d..f4603bf3b3 100644 --- a/docs/en/api/01-overview.md +++ b/docs/en/api/01-overview.md @@ -359,10 +359,6 @@ This catalog follows the routes actually mounted by the server. Each group headi | GET | `/api/v1/system/status` | System status | | POST | `/api/v1/system/wait` | Wait for background processing | | POST | `/api/v1/system/consistency` | Check filesystem and vector-index consistency | -| POST | `/api/v1/system/backend/sync-status` | Query backend synchronization status | -| POST | `/api/v1/system/backend/sync-retry` | Retry backend synchronization | -| GET | `/api/v1/system/sync/{sync_path}` | Path-form compatibility endpoint for synchronization status | -| POST | `/api/v1/system/sync/{sync_path}/retry` | Path-form compatibility endpoint for synchronization retry | ### [Resources](02-resources.md) and [Filesystem](03-filesystem.md) diff --git a/docs/en/api/07-system.md b/docs/en/api/07-system.md index 78a175fcc9..d87aefa607 100644 --- a/docs/en/api/07-system.md +++ b/docs/en/api/07-system.md @@ -1,6 +1,8 @@ # System Status -The OpenViking System API provides health, readiness, consistency, and multi-write backend synchronization status. Component observers and Prometheus metrics are documented separately. +The OpenViking System API provides health, readiness, consistency, and +background-processing status. Component observers and Prometheus metrics are +documented separately. ## API Reference @@ -417,99 +419,6 @@ ov system wait --timeout 60 --- -### backend_sync_status() - -Return multi-write backend synchronization status for a Viking URI subtree. This endpoint requires ROOT or ADMIN permission. - -**HTTP API** - -```http -POST /api/v1/system/backend/sync-status -Content-Type: application/json -``` - -```bash -curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \ - -H "Content-Type: application/json" \ - -H "X-API-Key: your-admin-key" \ - -d '{"uri":"viking://resources"}' -``` - -The URI-path form is also available: - -```http -GET /api/v1/system/sync/{sync_path} -``` - -**CLI** - -```bash -ov system backend sync-status viking://resources -``` - -**Response example** - -```json -{ - "status": "ok", - "result": { - "path": "viking://resources", - "entry_count": 12 - } -} -``` - -`result` is supplied by the active filesystem backend. `path` identifies the queried scope and `entry_count` is the number of sync records in that scope. A backend may add diagnostics such as pending or failed records. - -### backend_sync_retry() - -Retry incomplete multi-write backend synchronization work for a URI subtree. This endpoint requires ROOT or ADMIN permission. - -**HTTP API** - -```http -POST /api/v1/system/backend/sync-retry -Content-Type: application/json -``` - -```bash -curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \ - -H "Content-Type: application/json" \ - -H "X-API-Key: your-admin-key" \ - -d '{"uri":"viking://resources"}' -``` - -The URI-path form is: - -```http -POST /api/v1/system/sync/{sync_path}/retry -``` - -**CLI** - -```bash -ov system backend sync-retry viking://resources -``` - -**Response example** - -```json -{ - "status": "ok", - "result": { - "path": "viking://resources", - "retried": 2, - "failed": 0 - } -} -``` - -`retried` is the number of records rescheduled by this request, and `failed` is the number that could not be scheduled. A backend may include additional diagnostic fields. - -The public Python, TypeScript, and Go SDKs do not currently expose multi-write backend synchronization methods, so the sections above show only HTTP and CLI tabs. - ---- - ## Related Documentation diff --git a/docs/en/api/20-webdav.md b/docs/en/api/20-webdav.md index 161ba6d19c..9526ff464c 100644 --- a/docs/en/api/20-webdav.md +++ b/docs/en/api/20-webdav.md @@ -17,7 +17,7 @@ The adapter supports the following scope: - Resources only. Memories, skills, sessions, and other namespaces are not exposed. - Text-first writes. `PUT` currently accepts UTF-8 text content only. - WebDAV subset only. `OPTIONS`, `PROPFIND`, `GET`, `HEAD`, `PUT`, `DELETE`, `MKCOL`, and `MOVE` are supported. -- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, `.redirect.json`, and `.sync_log.json` are hidden from listings and cannot be accessed directly through WebDAV. +- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, and multi-write metadata are hidden from listings and cannot be accessed directly through WebDAV. Behavior notes: @@ -25,7 +25,7 @@ Behavior notes: - Replacing an existing file through WebDAV refreshes related semantics and vectors, same as `write()`. - `PUT` does not create parent collections automatically. Create missing directories with `MKCOL` first. - User-created dot-directories and dot-files remain visible unless they match one of the reserved internal filenames above. -- When multi-write storage is enabled, files redirected to a backup are still exposed through the filesystem APIs as normal files; internal redirect and sync metadata never become visible to callers. +- With multi-write enabled, WebDAV reads use the primary. Backup replication is asynchronous. ## API Reference diff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.md index ccedc225e8..b117329df0 100644 --- a/docs/en/concepts/05-storage.md +++ b/docs/en/concepts/05-storage.md @@ -54,10 +54,10 @@ AGFS provides POSIX-style file operations with multiple backend support. By default, AGFS uses a single backend for content storage. Once `storage.agfs.backups` is configured, OpenViking enters multi-write mode: -- Top-level `storage.agfs.backend` is the primary backend and remains the authoritative write target. -- `storage.agfs.backups.items[]` defines backup backends for replicas, migration, or read acceleration. +- Top-level `storage.agfs.backend` is the primary backend and the only direct read source. +- `storage.agfs.backups.items[]` defines asynchronously updated backup backends. - The Python SDK, HTTP API, and CLI filesystem interfaces stay unchanged. -- Multi-write uses `.redirect.json` and `.sync_log.json` internally to track redirect mappings and sync progress. These files are not visible to users. +- V2 metadata is partitioned and stored only on the primary. For the conceptual model, see [Multi-Write Storage](./14-multi-write-storage.md). For examples, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md). @@ -161,5 +161,5 @@ client.mv( - [Architecture Overview](./01-architecture.md) - System architecture - [Context Layers](./03-context-layers.md) - L0/L1/L2 model - [Viking URI](./04-viking-uri.md) - URI specification -- [Multi-Write Storage](./14-multi-write-storage.md) - Primary/backup roles, routing, and consistency +- [Multi-Write Storage](./14-multi-write-storage.md) - V2 primary and backup consistency - [Retrieval Mechanism](./07-retrieval.md) - Retrieval process details diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md index b6da7c6bb2..224b96a9f4 100644 --- a/docs/en/concepts/12-metrics.md +++ b/docs/en/concepts/12-metrics.md @@ -289,8 +289,12 @@ seconds, for example `123 ns = 0.000000123 seconds`. | `openviking_ragfs_cache_bytes_total` | Counter | `source` | bytes from backend/cache | | `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | cumulative get/put/delete duration | | `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved counts | -| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | background tasks, including the retry loop | -| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss selections | +| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | active flush runtime, `0` or `1` per mount | +| `openviking_ragfs_multiwrite_pending_events` | Gauge | none | submitted events that are queued, executing, or awaiting retry | +| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | final failures for `flush`, `checkpoint`, `catch_up`, or `gc` | +| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | sum of `tail - synced_seq` across accounts | +| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | mount count for the closed label `v2` | +| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | mount count for closed labels `migrating` and `stable` | These metrics have no `mount` or `account_id` label. Cache and multi-backend families are absent when those capabilities are not enabled. `status` is @@ -298,6 +302,10 @@ families are absent when those capabilities are not enabled. `status` is Histogram buckets cover all outcomes, with finite bounds from 0.0001 to 10 seconds. Python `get_stats()` retains its microsecond fields. +The protocol family always contains both closed-label series. One unreadable +or invalid protocol, account, or segment manifest rejects the +whole scrape snapshot instead of publishing partial zero lag. + ### Tasks and Task Tracker | Metric Family | Type | Common Labels | Meaning | diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.md index b5ffd735b8..356cab111c 100644 --- a/docs/en/concepts/14-multi-write-storage.md +++ b/docs/en/concepts/14-multi-write-storage.md @@ -1,134 +1,51 @@ # Multi-Write Storage -Multi-write storage lets OpenViking use one primary storage backend together with multiple backup backends under a unified filesystem abstraction. It supports replicas (including cross-region replicas), configurable read routing, and storage migration; it does not automatically promote a backup to primary. +V2 multi-write keeps one primary backend as the business source of truth and +replicates its files to configured backups. -From the API user's point of view, interfaces such as `read()`, `write()`, `ls()`, and `stat()` do not change. Multi-write logic lives inside RAGFS, so callers do not need to care which underlying backend ultimately stores a file. +## Read And Write Model -## Core Model +- All direct reads use the primary. Backups never serve application reads. +- A successful primary mutation returns immediately. +- Metadata persistence and backup application continue asynchronously. +- Backup lag does not change the result already returned for the primary. +- Multi-write metadata is stored only on the primary. -Multi-write storage consists of one primary backend and multiple backup backends: +This model is eventually consistent. A process crash between primary success +and event persistence can lose that replication event. Rebuild affected +backups from the primary after such a crash. -| Role | Configuration | Description | -| --- | --- | --- | -| primary | `storage.agfs.backend` | Authoritative write target and final read fallback | -| backup | `storage.agfs.backups.items[]` | Receives replicated writes and may optionally participate in reads | +## Metadata Model -If `backups` is not configured, OpenViking continues to use the original single-backend mode. +Each account has a fixed set of metadata partitions. Events are routed to one +partition, appended to a segment log, and consumed independently by each +backup. Checkpoints compact older history, and garbage collection removes +metadata that is no longer required. -## Write Path +`initial_partitions` controls the partition count when an account's metadata is +created. Existing accounts keep their persisted partition count and routes when +the configuration changes. Partition expansion and shrinking are not supported +in this release. -By default, writes land on the primary backend first and are then replicated to write-enabled backup backends. +## First Enable Import -```text -Client - -> OpenViking API - -> RAGFS MultiWrite - -> primary - -> backup1 / backup2 / ... -``` +When `.multiwrite.json` is absent, startup creates it as `V2/migrating`, +starts foreground V2 event submission, and imports the current primary files +in the background. Successful import changes the status to `stable`. Extra +old files on backups are not removed. -If a backup does not define `operations`, it participates in writes by default. This keeps cold-backup setup simple. +Old multi-write protocols and old migration fields are rejected. They are not +converted or interpreted. -## Sync Modes +## Operations -Multi-write supports two consistency modes. - -| Mode | Config value | Behavior | Suitable for | -| --- | --- | --- | --- | -| Async multi-write | `async` | Return as soon as the primary write succeeds; backups sync in the background | Low-latency writes, eventual consistency | -| Sync multi-write | `sync` | Wait for backup acknowledgements after the primary write succeeds | Stronger write confirmation when extra latency is acceptable | - -In async mode, backups may lag behind the primary; recovery depends on backend availability and retry outcomes. In sync mode, `write_ack_count` and `write_ack_timeout_ms` control how many backup acknowledgements are required and how long the system waits. - -If sync mode fails to reach the required acknowledgement count, the call returns an error even though the primary may already have committed; that write is not automatically rolled back. Backups that did not acknowledge are still retried in the background; to inspect sync state, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md). - -## Read Path - -Reads do not hit every backup by default. Only backups that explicitly declare the `read` operation join the read route. - -Read order: - -```text -1. Read-enabled backups in ascending priority order -2. Fallback to the primary backend -3. If the file is redirected, access the redirect target -4. Return NotFound if the file is still missing -``` - -This avoids having cold-backup nodes participate in reads by default and reduces the risk of serving stale data. - -## Redirect - -Redirect means "certain files are not written to the primary backend and are written to a specified backup instead." - -Common cases: - -- Large files go to object storage. -- Specific file extensions go to a dedicated backend. -- The primary backend keeps standard content, while special files live elsewhere. - -Redirect policies are configured on the primary backend. When a file matches a policy, OpenViking records the mapping in internal metadata. Calls such as `ls()`, `stat()`, and `read()` still expose a normal filesystem view. - -## Exclude - -Exclude means "a specific backup backend does not receive matching files." - -Common cases: - -- A memory or cache backend should not keep large files. -- One backup only stores text resources. -- A lower-cost backend excludes temporary or oversized files. - -Exclude policies are configured on each backup and affect only whether that backup receives a write. - -## Internal Metadata - -Multi-write uses two internal metadata files: - -| File | Purpose | -| --- | --- | -| `.redirect.json` | Records which backend stores each redirected file | -| `.sync_log.json` | Records per-file sync version and backup acknowledgement progress | - -These files are hidden from normal users, do not appear in standard directory listings, and should not be read or written through public APIs. - -If the primary backend enables at-rest encryption, these internal metadata files follow the same encryption policy. - -## Encryption Relationship - -Multi-write does not change OpenViking's transparent encryption model. - -Rules: - -- The Python layer and public APIs stay unaware of encryption details. -- The primary backend must be encrypted when global encryption is enabled. -- Each backup may decide independently whether to enable encryption. -- Internal metadata must go through the primary backend's encryption path. - -That means enabling multi-write does not change how clients call the system; encryption remains a configuration concern per backend. - -## Relationship with OVPack - -Multi-write only handles new writes after it is enabled. It does not automatically sync historical files that already exist in the primary backend. - -Recommended migration flow: - -1. Pause writes or arrange a verifiable incremental catch-up window, then migrate existing data with OVPack or another controlled process. -2. Validate the target backend data. -3. Enable the multi-write configuration. -4. Catch up changes made during migration and verify sync state before resuming writes; multi-write replicates subsequent additions and updates. - -## Limitations - -- In async mode, backups may lag; reads from them cannot assume the latest write is visible. -- Historical files that existed before multi-write was enabled need a separate migration or backfill process. -- Redirected files rely on internal metadata to reconstruct the directory view. -- Concurrent writes from multiple processes to the same primary backend still need future distributed metadata locking. -- Hot directories may update internal metadata frequently and can introduce additional write amplification. +V2 does not expose synchronization status or manual retry APIs or CLI +commands. Use the multi-write Prometheus metrics to observe queue depth, +worker failures, backup lag, protocol version, and protocol status. ## Related Documents - [Storage Architecture](./05-storage.md) +- [Metrics](./12-metrics.md) - [Configuration Guide](../guides/01-configuration.md) - [Multi-Write Storage Guide](../guides/13-multi-write-storage.md) -- [OVPack Import and Export](../guides/09-ovpack.md) diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 0d47386b31..70ac61e233 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1300,7 +1300,6 @@ Storage configuration for context data, including file storage (RAGFS) and vecto | `backend` | str | `"local"`, `"s3"`, or `"memory"` | `"local"` | | `timeout` | float | Request timeout in seconds | `10.0` | | `backups` | object | Multi-write storage configuration. When set, the top-level `backend` acts as the primary backend and `backups.items[]` defines backup backends | `null` | -| `redirects` | array | File redirect policies for multi-write storage. Matching files are written to the specified backup instead of the primary backend | `[]` | | `queuefs` | object | QueueFS configuration. Controls the namespace mode, backend, and runtime options for `/queue` | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` | | `queue_db_path` | str (optional) | Legacy compatibility field for QueueFS sqlite DB path. Superseded by `storage.agfs.queuefs.db_path`. Defaults to `{storage.workspace}/_system/queue/queue.db` when not set. Useful when the workspace volume does not support sqlite (e.g. some network filesystems) | `null` | | `s3` | object | S3 backend configuration (when backend is 's3') | - | @@ -1314,7 +1313,8 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro ##### Multi-Write Storage Configuration -`storage.agfs.backups` enables multi-write storage. If it is not configured, OpenViking stays in single-backend mode. +`storage.agfs.backups` enables asynchronous V2 multi-write storage. The +top-level backend remains the only direct read source. ```json { @@ -1322,26 +1322,16 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro "workspace": "./data", "agfs": { "backend": "local", - "redirects": [ - { - "type": "FileExtensionPolicy", - "extensions": ["(pdf|ppt|zip)"], - "target": ["s3-backup"] - } - ], "backups": { - "sync_type": "async", + "initial_partitions": 16, + "checkpoint_interval_secs": 86400, + "provider": "filesystem", "items": [ { - "name": "s3-backup", - "backend": "s3", - "s3": { - "bucket": "openviking-backup", - "region": "cn-beijing", - "endpoint": "https://tos-s3-cn-beijing.volces.com", - "access_key": "your-ak", - "secret_key": "your-sk", - "prefix": "multi-write" + "name": "local-backup", + "backend": "local", + "params": { + "workspace": "./data/backup" } } ] @@ -1355,36 +1345,14 @@ Common `backups` fields: | Parameter | Type | Description | Default | |-----------|------|-------------|---------| -| `sync_type` | str | Multi-write sync mode. Supports `"async"` or `"sync"` | `"async"` | -| `write_ack_count` | int | Number of backup acknowledgements required before a `sync` write returns | all backups | -| `write_ack_timeout_ms` | int | Timeout in milliseconds while waiting for backup acknowledgements in `sync` mode | `null` | -| `write_concurrency` | int | Maximum async backup write concurrency | `null` | -| `items` | array | Backup backend list. Each item reuses normal backend configuration and adds fields such as `name`, `operations`, `excludes`, and `encryption` | `[]` | - -Common `redirects` fields: - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `type` | str | Policy type. Supports `"FileExtensionPolicy"` or `"FileOverSizePolicy"` | required | -| `extensions` | array | Extension regex list used by `FileExtensionPolicy`, for example `["(pdf\\|ppt)"]` | `[]` | -| `max_size_mb` | int | File size threshold in MB used by `FileOverSizePolicy` | `null` | -| `target` | array | Backup `name` list that receives matched files | required | - -File-size redirect example: - -```json -{ - "type": "FileOverSizePolicy", - "max_size_mb": 100, - "target": ["s3-backup"] -} -``` - -Notes: +| `initial_partitions` | int | Initial metadata partitions; changes do not affect existing accounts | `16` | +| `checkpoint_interval_secs` | int | Checkpoint discovery interval; minimum `60` | `86400` | +| `provider` | str | Metadata provider: `filesystem` or top-level CacheRuntime-backed `cache` | `"filesystem"` | +| `items` | array | Backups with `name`, `backend`, `params`, and optional `encryption` | `[]` | -- `redirects` is configured at top-level `storage.agfs` and defines redirect policies for the primary backend. -- `target` must reference an existing backup `name` from `backups.items[]`. -- Files matched by redirect still appear as normal readable and listable files through the filesystem APIs. +The Python configuration layer still accepts legacy sync, retry, operations, +excludes, and redirects fields. It warns and removes them before calling Rust; +the fields have no effect. See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples. @@ -1452,9 +1420,9 @@ The top-level `cache` section is a sibling of `storage`. Its public shape is Pro } ``` -The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, or when PathLock selects `provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened. +The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, PathLock selects `provider=cache`, or multi-write selects `backups.provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened. -This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, and `storage.agfs.queuefs.redis` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"` or `queuefs.backend="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`. +This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, `storage.agfs.queuefs.redis`, and `storage.agfs.backups.provider="redis"` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"`, `queuefs.backend="cache"`, or `backups.provider="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`. ##### QueueFS Configuration diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.md index c10ad8926e..ea971ce81e 100644 --- a/docs/en/guides/08-encryption.md +++ b/docs/en/guides/08-encryption.md @@ -21,7 +21,7 @@ Rules: - When global `encryption.enabled=true`, the primary backend must be encrypted. - Each backup backend may control its own encryption through `encryption.enabled`. -- Multi-write internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy. +- V2 multi-write metadata is stored on the raw primary and bypasses `EncryptionWrappedFS`. - OpenViking does not expose and does not need public encryption APIs for operating on these internal files. See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more multi-write configuration details. diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.md index eab78f7a77..8b117348f2 100644 --- a/docs/en/guides/09-ovpack.md +++ b/docs/en/guides/09-ovpack.md @@ -38,26 +38,13 @@ control files, and `.relations.json` are outside the OVPack migration scope. ## Working with Multi-Write Storage -Multi-write storage only replicates writes that happen after it is enabled. It -does not automatically copy historical files that already existed before -`storage.agfs.backups` was turned on. +V2 startup imports current V1 primary files into partitioned metadata and +removes V1 sidecars. It does not remove extra old files from backups. Pause +all instances and take a complete backup before the first V2 startup. -To seed the primary and replicas together, enable multi-write on the **empty target environment before restoring** the package: - -1. Pause application writes and export or back up each source account. -2. Configure the target primary and backup backends, including their write policies, then start the target server and provision its restore identity. -3. Restore or import through that server so historical content passes through the configured write fanout. -4. Check synchronization status for every restored scope and verify the expected files in each replica before switching traffic. Asynchronous replication can still be pending after restore returns. -5. Resume writes after validating content and index integrity. Keep the source and backup until verification is complete. - -For example, using the target account's admin key: - -```bash -ov system backend sync-status viking://resources -ov system backend sync-status viking://user -``` - -Restoring first and enabling backups afterwards only replicates subsequent writes; it does not seed historical content into replicas. +Once import starts, in-place rollback is unsupported. Restore the complete +pre-import backup to return to V1. After startup, verify primary content with +normal file APIs and monitor backup lag through Prometheus metrics. For more details, see the [Multi-Write Storage Guide](./13-multi-write-storage.md). diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.md index 2c6c8f3188..4a30c18abf 100644 --- a/docs/en/guides/13-multi-write-storage.md +++ b/docs/en/guides/13-multi-write-storage.md @@ -1,20 +1,10 @@ # Multi-Write Storage Guide -This guide explains how to configure multi-write storage in OpenViking. Multi-write storage lets one primary backend replicate writes to multiple backup backends for replica storage (including cross-region replicas), optional read routing, and migration. It does not automatically promote a backup when the primary fails. - -Multi-write lives inside RAGFS. The Python SDK, HTTP API, and CLI usage remain unchanged. - -## Prerequisites - -- You already have a working `ov.conf`. -- The primary backend has been verified to read and write correctly. -- If you plan to use S3-compatible storage, prepare the bucket, endpoint, and access credentials first. -- For existing data, follow the [OVPack multi-write migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups alone does not copy historical files. +V2 multi-write returns after the primary succeeds and updates metadata and +backups asynchronously. The primary is the only direct read source. ## Minimal Configuration -Merge these excerpts into your existing model/server configuration. The following example uses a local directory as the primary backend and replicates writes to another local directory. - ```json { "storage": { @@ -22,13 +12,15 @@ Merge these excerpts into your existing model/server configuration. The followin "agfs": { "backend": "local", "backups": { - "sync_type": "async", + "initial_partitions": 16, + "checkpoint_interval_secs": 86400, + "provider": "filesystem", "items": [ { "name": "local-backup", "backend": "local", - "local": { - "workspace": "./data/backup" + "params": { + "workspace": "./data/backup" } } ] @@ -38,385 +30,62 @@ Merge these excerpts into your existing model/server configuration. The followin } ``` -Notes: - -- The top-level `backend` is the primary backend. -- `backups.items[]` is the backup backend list. -- `name` is the stable identity of a backup; later sync metadata refers to it. -- A backup with `backend = "local"` uses `local.workspace` to point to its local directory. -- If `sync_type` is omitted, treat it as async by default. - -## Configuring Multiple Backups +The three top-level V2 settings are: -You can configure more than one backup. The following example writes to both a local replica and S3-compatible object storage. - -```json -{ - "storage": { - "workspace": "./data", - "agfs": { - "backend": "local", - "backups": { - "sync_type": "async", - "items": [ - { - "name": "local-az2", - "backend": "local", - "local": { - "workspace": "./data/local-az2" - } - }, - { - "name": "object-store", - "backend": "s3", - "s3": { - "bucket": "openviking-backup", - "region": "us-east-1", - "endpoint": "https://s3.example.com", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "prefix": "openviking", - "directory_marker_mode": "none" - } - } - ] - } - } - } -} -``` - -Recommendations: - -- Do not use unstable hostnames or temporary IDs for `name`. -- The backup path or bucket should not point to the same physical location as the primary backend. -- Changing a backup `name` affects historical sync metadata recognition, so treat that as a production change. - -### S3-Compatible Storage Notes - -Match these `s3` fields to the endpoint and object-store configuration: - -| Field | Required? | Description | +| Field | Default | Meaning | | --- | --- | --- | -| `use_path_style` | Optional; default `true` | Use path-style URLs (`http://host/bucket/key`); select the style your endpoint supports. | -| `directory_marker_mode` | Yes for backup items | `none` creates no marker, `empty` creates a zero-byte marker, and `nonempty` creates a marker with content. Set it explicitly in each S3 backup item; if omitted, startup fails with `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`. | -| `use_ssl` | Optional | Set to `false` for HTTP endpoints (e.g. `http://localhost:9000`). | - -**Minimal S3-compatible example (RustFS/MinIO):** - -```json -{ - "name": "s3-backup", - "backend": "s3", - "s3": { - "bucket": "my-bucket", - "endpoint": "http://localhost:9000", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "prefix": "openviking", - "use_ssl": false, - "use_path_style": true, - "directory_marker_mode": "none" - } -} -``` - -The example uses `none`, which suits S3-compatible services that do not use directory markers (RustFS, MinIO, Ceph, etc.). Backup items do not get a default for this field. The top-level `storage` S3 configuration defaults to `empty`. - -### Docker Networking for S3 Backup - -When running OpenViking in Docker with an S3 backup on the same host: - -- **Linux Docker**: Use `--network host` or the host's LAN IP. Docker bridge network can reach the host's LAN via gateway IP (e.g. `172.17.0.1:9000`). -- **macOS/Windows Docker Desktop**: Use `host.docker.internal` to reach the host from the container. Host networking is available as an opt-in feature in Docker Desktop 4.34 and later; see [Docker host networking](https://docs.docker.com/engine/network/drivers/host/). - -If the server crashes silently on startup with S3 backup enabled, check the Docker networking first. The RAGFS Rust binding will produce `dispatch failure` if the S3 endpoint is unreachable from inside the container. - -## Choosing a Sync Mode - -### Async Mode - -Async mode fits most deployments. - -```json -{ - "backups": { - "sync_type": "async", - "items": [] - } -} -``` - -Characteristics: - -- Returns immediately after the primary write succeeds. -- Backup writes run in the background. -- Low write latency. -- Lag has no fixed bound while a backend is unavailable; monitor synchronization and recovery. - -Suitable for: - -- Write throughput first. -- Backups mainly used for disaster recovery. -- Eventual consistency is acceptable. - -### Sync Mode - -Sync mode waits for backup acknowledgements. - -```json -{ - "backups": { - "sync_type": "sync", - "write_ack_count": 1, - "write_ack_timeout_ms": 5000, - "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}] - } -} -``` - -Parameters: - -| Parameter | Description | -| --- | --- | -| `write_ack_count` | Minimum number of backup acknowledgements required before the write returns | -| `write_ack_timeout_ms` | Timeout in milliseconds while waiting for backup acknowledgements | - -Characteristics: - -- Stronger write confirmation. -- Write latency depends on backup responsiveness. -- Unconfirmed backups continue to be retried in the background. -- The client may still see an error after the primary write has already succeeded when the required backup acknowledgements are not met. - -Suitable for: - -- Narrowing the confirmation window between the primary and backups. -- Backup latency is predictable. -- The caller can accept the extra latency of synchronous writes. - -## Configuring Read Acceleration - -Backups do not participate in reads by default. To let a backup serve reads, explicitly configure `operations`. - -```json -{ - "name": "cache-backend", - "backend": "memfs", - "operations": [ - {"operation": "write", "priority": 0}, - { - "operation": "read", - "priority": 10 - } - ] -} -``` - -Read priority rules: - -- Lower `priority` values are tried first. -- Only backups with `read` configured join the read route. -- The primary backend always remains the final fallback. -- Cold-backup nodes usually should not be read-enabled. - -If a backup defines only `read` but not `write`, it does not receive normal multi-write replication. Use that only when you explicitly control how the backend gets its data. +| `initial_partitions` | `16` | Metadata partitions created for an account | +| `checkpoint_interval_secs` | `86400` | Checkpoint discovery interval; minimum `60` | +| `provider` | `"filesystem"` | Metadata segment provider: `filesystem` or `cache` | -## Redirect Configuration +Each `items[]` entry needs `name`, `backend`, and backend-specific `params`. +`encryption` is optional. Backup names must be unique and cannot be +`primary`. -Redirect sends matching files to a specified backup instead of the primary backend. +When `provider` is `cache`, multi-write reuses the top-level CacheRuntime. +The top-level `cache.provider` selects the implementation; multi-write has no +separate Cache Provider connection. -Redirect by file extension: +The Python configuration layer still accepts legacy V1 sync, acknowledgement, +retry, operation, exclude, and redirect fields. It warns and removes them +before calling Rust; the fields have no effect. -```json -{ - "storage": { - "agfs": { - "backend": "local", - "redirects": [ - { - "type": "FileExtensionPolicy", - "extensions": ["(pdf|ppt|zip)"], - "target": ["object-store"] - } - ], - "backups": { - "items": [ - { - "name": "object-store", - "backend": "s3", - "s3": { - "bucket": "openviking-large-files", - "endpoint": "https://s3.example.com" - } - } - ] - } - } - } -} -``` - -Redirect by file size: - -```json -{ - "type": "FileOverSizePolicy", - "max_size_mb": 100, - "target": ["object-store"] -} -``` - -Notes: - -- `target` must reference an existing backup `name`. -- Redirected files still appear as normal readable, listable, and queryable files through the public APIs. -- Redirect mappings are stored in internal metadata on the primary backend. - -## Exclude Configuration - -Exclude makes one backup skip matching files. - -```json -{ - "name": "cache-backend", - "backend": "memfs", - "excludes": [ - { - "type": "FileOverSizePolicy", - "max_size_mb": 50 - }, - { - "type": "FileExtensionPolicy", - "extensions": ["(mp4|zip)"] - } - ] -} -``` - -Common uses: - -- Exclude large files from cache backends. -- Exclude file types that do not need to be preserved on a lower-cost backup. -- Keep one backup focused on text or configuration resources only. - -If the target backup of a redirect also excludes the same file, the configuration is self-contradictory. Fix the configuration instead of expecting the system to guess another target. - -## Encryption Configuration - -Multi-write storage reuses OpenViking's transparent at-rest encryption. - -Example with global encryption enabled: - -```json -{ - "encryption": { - "enabled": true, - "provider": "local", - "local": { - "key_file": "~/.openviking/master.key" - } - }, - "storage": { - "workspace": "./data", - "agfs": { - "backend": "local", - "backups": { - "items": [ - { - "name": "plain-cache", - "backend": "memfs", - "encryption": { - "enabled": false - } - }, - { - "name": "encrypted-backup", - "backend": "local", - "local": { - "workspace": "./data/encrypted-backup" - }, - "encryption": { - "enabled": true - } - } - ] - } - } - } -} -``` - -Rules: - -- When global `encryption.enabled=true`, the primary backend must be encrypted. -- Each backup may independently control encryption through `encryption.enabled`. -- The Python SDK, HTTP API, and CLI do not need to handle encryption or decryption. -- Internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy. +## Runtime Behavior -## Migrating Existing Data - -Multi-write only replicates writes that happen after it is enabled. It does not automatically copy historical files. - -For OVPack migration, follow [Working with Multi-Write Storage](./09-ovpack.md#working-with-multi-write-storage): configure multi-write on the clean target before restoring through that server, then verify each replica before resuming writes. That guide also covers target account initialization and restore conflict handling. - -## Verifying the Configuration - -Before startup, it is recommended to run: - -```bash -openviking-server doctor -``` - -After startup, verify with ordinary file APIs: - -```bash -ov write viking://resources/multiwrite-check.txt \ - --content "multi-write check" - -ov read viking://resources/multiwrite-check.txt -``` - -If you use a local backup, you can also inspect the backup directory directly. Also run `ov system backend sync-status viking://resources` and check every intended replica. A successful ordinary read may come only from the primary and does not prove replication. - -## FAQ - -### Why is a backup not serving reads? - -Backups are write-only by default. To make a backup serve reads, configure: - -```json -{ - "operations": [ - {"operation": "write", "priority": 0}, - { - "operation": "read", - "priority": 10 - } - ] -} -``` +- The primary is the only direct read source and business source of truth. +- Primary success returns without waiting for metadata or backups. +- Multi-write metadata is stored only on the primary. +- Backups converge in the background. +- There is no synchronization status or manual retry API or CLI command. -### Why do historical files not appear in the backup after enabling multi-write? +A crash between primary success and event persistence can lose one replication +event. Rebuild an affected backup from the primary. -Multi-write only handles new writes after it is enabled. For historical data, follow the [OVPack migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups does not backfill it. +## Partition Count -### Can async mode guarantee that the newest data is immediately readable from a backup? +`initial_partitions` applies only when an account's partition manifest is first +created. Existing accounts continue to use their persisted partition count and +routes after configuration changes. Partition expansion and shrinking are not +supported in this release. -No. Background retries need a functioning backend to catch up. A stale but successful backup read does not fall back to the primary; route reads to the primary when current data is required. +## First Enable And Rollback -### Will internal metadata files appear in normal user listings? +When `.multiwrite.json` is absent, the mount starts foreground V2 event +submission and imports current primary files in the background. Extra old +files on backups are not removed. -No. `.redirect.json` and `.sync_log.json` are internal files and are hidden from ordinary directory listings. +Old multi-write protocols and old migration fields are rejected. Take a +complete backup before enabling multi-write. Restore that backup before +rolling back the binary. -### If sync mode returns a failure, does that mean the primary backend definitely did not write the data? +## Verification -No. The primary write may already have succeeded while the required backup acknowledgements were not met. In that case the client can see a failure even though the data already exists on the primary backend, and lagging backups will continue to be repaired in the background. +Use normal file APIs to verify primary reads and writes. Use the metrics in +[Metrics](../concepts/12-metrics.md) to monitor queue depth, worker failures, +backup lag, protocol version, and protocol status. ## Related Documents - [Multi-Write Storage](../concepts/14-multi-write-storage.md) -- [Storage Architecture](../concepts/05-storage.md) - [Configuration Guide](./01-configuration.md) -- [Encryption Guide](./08-encryption.md) - [OVPack Import and Export](./09-ovpack.md) diff --git a/docs/scripts/check-api-reference.mjs b/docs/scripts/check-api-reference.mjs index 6a1a4432ef..d83dcb2124 100644 --- a/docs/scripts/check-api-reference.mjs +++ b/docs/scripts/check-api-reference.mjs @@ -178,11 +178,9 @@ function findRoute(method, documentedPath) { const candidateMethod = candidate.slice(0, separator) const candidatePath = candidate.slice(separator + 1) if (candidateMethod !== method) continue - const placeholderPattern = - candidatePath.startsWith('/webdav/resources/{}') || - candidatePath.startsWith('/api/v1/system/sync/{}') - ? '.+' - : '[^/]+' + const placeholderPattern = candidatePath.startsWith('/webdav/resources/{}') + ? '.+' + : '[^/]+' const pattern = candidatePath .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') .replace(/\\\{\\\}/g, placeholderPattern) @@ -351,10 +349,7 @@ function routeIsCovered(route, coverage) { .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') .replace( /\\\{\\\}/g, - routePath.startsWith('/webdav/resources/{}') || - routePath.startsWith('/api/v1/system/sync/{}') - ? '.+' - : '[^/]+' + routePath.startsWith('/webdav/resources/{}') ? '.+' : '[^/]+' ) for (const reference of coverage) { const referenceSeparator = reference.indexOf(' ') diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md index 39d6406b73..8e278afd1f 100644 --- a/docs/zh/api/01-overview.md +++ b/docs/zh/api/01-overview.md @@ -354,10 +354,6 @@ JSON 输出 - 错误： | GET | `/api/v1/system/status` | 系统状态 | | POST | `/api/v1/system/wait` | 等待后台处理完成 | | POST | `/api/v1/system/consistency` | 文件系统与向量索引一致性检查 | -| POST | `/api/v1/system/backend/sync-status` | 查询后端同步状态 | -| POST | `/api/v1/system/backend/sync-retry` | 重试后端同步 | -| GET | `/api/v1/system/sync/{sync_path}` | 路径形式的同步状态兼容接口 | -| POST | `/api/v1/system/sync/{sync_path}/retry` | 路径形式的同步重试兼容接口 | ### [资源](02-resources.md)与[文件系统](03-filesystem.md) diff --git a/docs/zh/api/07-system.md b/docs/zh/api/07-system.md index 9281b3fb1f..420fac892e 100644 --- a/docs/zh/api/07-system.md +++ b/docs/zh/api/07-system.md @@ -1,6 +1,7 @@ # 系统状态 -OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和主备后端同步状态。组件级观测和 Prometheus 指标分别提供独立文档。 +OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和后台处理状态。 +组件级观测和 Prometheus 指标分别提供独立文档。 ## API 参考 @@ -409,99 +410,6 @@ ov system wait --timeout 60 --- -### backend_sync_status() - -查询指定 Viking URI 子树在主备存储后端之间的同步状态。该接口要求 ROOT 或 ADMIN 权限。 - -**HTTP API** - -```http -POST /api/v1/system/backend/sync-status -Content-Type: application/json -``` - -```bash -curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \ - -H "Content-Type: application/json" \ - -H "X-API-Key: your-admin-key" \ - -d '{"uri":"viking://resources"}' -``` - -也可以使用 URI 路径形式： - -```http -GET /api/v1/system/sync/{sync_path} -``` - -**CLI** - -```bash -ov system backend sync-status viking://resources -``` - -**响应示例** - -```json -{ - "status": "ok", - "result": { - "path": "viking://resources", - "entry_count": 12 - } -} -``` - -`result` 由当前文件系统后端返回；`path` 标识查询范围，`entry_count` 表示该范围内的同步记录数。具体后端可能附加待同步、失败记录等诊断字段。 - -### backend_sync_retry() - -重试指定 URI 子树中尚未完成的主备后端同步工作。该接口要求 ROOT 或 ADMIN 权限。 - -**HTTP API** - -```http -POST /api/v1/system/backend/sync-retry -Content-Type: application/json -``` - -```bash -curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \ - -H "Content-Type: application/json" \ - -H "X-API-Key: your-admin-key" \ - -d '{"uri":"viking://resources"}' -``` - -URI 路径形式为： - -```http -POST /api/v1/system/sync/{sync_path}/retry -``` - -**CLI** - -```bash -ov system backend sync-retry viking://resources -``` - -**响应示例** - -```json -{ - "status": "ok", - "result": { - "path": "viking://resources", - "retried": 2, - "failed": 0 - } -} -``` - -`retried` 是本次重新调度的记录数，`failed` 是重试调度失败的记录数；具体后端可能附加额外诊断字段。 - -公共 Python、TypeScript 和 Go SDK 当前没有主备后端同步方法，因此以上小节只展示 HTTP 和 CLI Tab。 - ---- - ## 相关文档 diff --git a/docs/zh/api/20-webdav.md b/docs/zh/api/20-webdav.md index 215d1b65cd..d6cf424bac 100644 --- a/docs/zh/api/20-webdav.md +++ b/docs/zh/api/20-webdav.md @@ -17,7 +17,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层 - 仅开放 `resources` 命名空间，不暴露 memories、skills、sessions 等其他空间。 - 以文本写入为主，当前 `PUT` 只接受 UTF-8 文本内容。 - 只实现一小部分 WebDAV 方法：`OPTIONS`、`PROPFIND`、`GET`、`HEAD`、`PUT`、`DELETE`、`MKCOL`、`MOVE`。 -- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock`、`.redirect.json`、`.sync_log.json` 这些派生或内部文件不会出现在 WebDAV 列表中，也不能被直接访问。 +- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock` 和多写 metadata 不会出现在 WebDAV 列表中，也不能被直接访问。 行为说明： @@ -25,7 +25,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层 - 通过 WebDAV 覆盖已有文件时，会像 `write()` 一样刷新相关语义和向量。 - `PUT` 不会自动创建父目录。缺失的目录需要先用 `MKCOL` 创建。 - 用户自己创建的点目录或点文件仍然可见，只有上面列出的保留内部文件名会被隐藏。 -- 启用主备存储时，被 redirect 到 backup 的文件仍会通过文件系统 API 呈现为普通文件；内部 redirect 和同步元数据不会暴露给调用方。 +- 启用多写时，WebDAV 读取只访问 primary，backup 在后台异步复制。 ## API 参考 diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.md index 710c964e13..1ab4eeae48 100644 --- a/docs/zh/concepts/05-storage.md +++ b/docs/zh/concepts/05-storage.md @@ -54,10 +54,10 @@ AGFS 提供 POSIX 风格的文件操作，支持多种后端。 默认情况下，AGFS 使用一个后端作为内容存储。配置 `storage.agfs.backups` 后，OpenViking 会启用主备模式： -- 顶层 `storage.agfs.backend` 是 primary，作为权威写入目标。 -- `storage.agfs.backups.items[]` 是 backup，用于副本、迁移或读加速。 +- 顶层 `storage.agfs.backend` 是 primary，也是唯一直接读取来源。 +- `storage.agfs.backups.items[]` 是后台异步更新的 backup。 - Python SDK、HTTP API 和 CLI 的文件系统接口保持不变。 -- 主备存储内部使用 `.redirect.json` 和 `.sync_log.json` 维护 redirect 映射与同步进度，这些文件对用户不可见。 +- V2 metadata 分区存储，并且只存储在 primary。 更多概念说明见 [主备存储](./14-multi-write-storage.md)，配置示例见 [主备存储指南](../guides/13-multi-write-storage.md)。 @@ -158,5 +158,5 @@ client.mv( - [架构概述](./01-architecture.md) - 系统整体架构 - [上下文层级](./03-context-layers.md) - L0/L1/L2 模型 - [Viking URI](./04-viking-uri.md) - URI 规范 -- [主备存储](./14-multi-write-storage.md) - primary/backup、主备路由与一致性 +- [多写存储](./14-multi-write-storage.md) - V2 primary 与 backup 一致性 - [检索机制](./07-retrieval.md) - 检索流程详解 diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md index 0ccd621fa2..29c021a127 100644 --- a/docs/zh/concepts/12-metrics.md +++ b/docs/zh/concepts/12-metrics.md @@ -288,8 +288,12 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba | `openviking_ragfs_cache_bytes_total` | Counter | `source` | 来自后端和缓存的字节数 | | `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | get/put/delete 累计耗时 | | `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved 次数 | -| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 后台任务数，包含重试循环 | -| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss 路由选择次数 | +| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 每个 mount 的活跃 flush runtime，值为 `0` 或 `1` | +| `openviking_ragfs_multiwrite_pending_events` | Gauge | 无 | 已提交且正在排队、执行中或重试中的事件数 | +| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | `flush`、`checkpoint`、`catch_up`、`gc` 的最终失败次数 | +| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | 所有 account 的 `tail - synced_seq` 之和 | +| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | 闭集标签 `v2` 对应的 mount 数 | +| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | 闭集标签 `migrating`、`stable` 对应的 mount 数 | 这些指标不带 `mount` 和 `account_id` 标签。未启用 Cache 或 multi-backend 时，不生成对应指标族。`status` 为 `success` 或 `error`。 @@ -297,6 +301,9 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba Histogram 包含全部操作结果，有限桶边界范围为 0.0001 至 10 秒。 Python `get_stats()` 保留原有微秒字段。 +protocol 指标始终包含闭集中的两条 series。任何 protocol、account 或 segment +manifest 无法读取或校验时，本次完整 snapshot 失败，不会发布部分 lag=0。 + ### 任务与 Task Tracker | 指标族 | 类型 | 常见标签 | 含义 | diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.md index 5464b5bb12..d7d0dbdc3a 100644 --- a/docs/zh/concepts/14-multi-write-storage.md +++ b/docs/zh/concepts/14-multi-write-storage.md @@ -1,134 +1,45 @@ # 主备存储 -主备存储（multi-write）让 OpenViking 在一个统一的文件系统抽象下，同时使用一个主存储和多个备份存储。它可用于保存副本（包括跨地域副本）、配置读路由和迁移存储；不会自动把 backup 提升为 primary。 +V2 多写以一个 primary backend 作为业务真相，并把文件复制到配置的 backup。 -从 API 使用者视角看，`read()`、`write()`、`ls()`、`stat()` 等接口不变。主备存储逻辑位于 RAGFS 内部，调用方不需要关心文件最终落在哪个底层后端。 +## 读写模型 -## 核心模型 +- 所有直接读取只访问 primary。backup 不参与业务读取。 +- primary 变更成功后立即返回。 +- metadata 持久化和 backup 应用在后台异步执行。 +- backup 落后不会改变 primary 已返回的结果。 +- 多写 metadata 只存储在 primary。 -主备存储由一个 primary 和多个 backup 组成： +该模型是最终一致。进程可能在 primary 成功和事件持久化之间崩溃，导致该次 +复制事件丢失。发生此情况后，需要从 primary 重建受影响的 backup。 -| 角色 | 配置位置 | 说明 | -| --- | --- | --- | -| primary | `storage.agfs.backend` | 权威写入目标，也是读取兜底 | -| backup | `storage.agfs.backups.items[]` | 接收复制写入，可选参与读取 | +## Metadata 模型 -没有配置 `backups` 时，OpenViking 继续使用原有单后端模式。 +每个 account 使用固定数量的 metadata partition。事件路由到一个 partition， +追加到 segment log，再由每个 backup 独立消费。Checkpoint 压缩旧历史，GC +清理不再需要的 metadata。 -## 写入路径 +`initial_partitions` 仅在 account metadata 首次创建时决定 partition 数量。 +已有 account 始终使用持久化的 partition 数量和 routes，修改配置不会改变 +现有布局。本期不支持扩容或缩容。 -默认情况下，写入先落到 primary，再复制到 write-enabled backup。 +## 首次启用导入 -```text -Client - -> OpenViking API - -> RAGFS MultiWrite - -> primary - -> backup1 / backup2 / ... -``` +`.multiwrite.json` 不存在时，启动流程创建 `V2/migrating` 状态，先启用前台 +V2 事件提交，再在后台导入 primary 当前文件。导入成功后切换为 `stable`。 +导入不会删除 backup 中多余的旧文件。 -backup 未配置 `operations` 时默认参与写入。这样可以用最少配置得到冷备能力。 +旧多写协议和旧迁移字段直接拒绝，不执行转换或解释。 -## 同步模式 +## 运维 -主备存储支持两种一致性模式。 - -| 模式 | 配置值 | 行为 | 适用场景 | -| --- | --- | --- | --- | -| 异步复制 | `async` | primary 写成功后立即返回，backup 后台同步 | 低延迟写入、最终一致 | -| 同步复制 | `sync` | primary 写成功后等待 backup 确认 | 更强写入确认、可接受额外延迟 | - -异步模式下，backup 可能落后于 primary，恢复时间取决于后端可用性和重试结果。同步模式下，可以通过 `write_ack_count` 和 `write_ack_timeout_ms` 控制需要等待多少 backup 确认，以及等待多久。 - -同步模式未达到所需确认数时会返回错误，但 primary 可能已经写入成功，不会因此自动回滚。未确认的 backup 仍会在后台重试；同步状态的检查方法见[主备存储指南](../guides/13-multi-write-storage.md)。 - -## 读取路径 - -读取不会默认访问所有 backup。只有显式声明 `read` 操作的 backup 才会进入读路由。 - -读取顺序如下： - -```text -1. 按 priority 升序访问 read-enabled backup -2. 回退到 primary -3. 如果文件被 redirect，则访问 redirect target -4. 仍未命中则返回 NotFound -``` - -这种设计避免冷备节点默认参与读取，降低读到旧数据的风险。 - -## Redirect - -Redirect 表示“某些文件不写入 primary，而是写入指定 backup”。 - -常见用途： - -- 大文件进入对象存储。 -- 特定后缀文件进入专门 backend。 -- 主存储只保存常规内容，特殊文件由其他 backend 保存。 - -Redirect 策略配置在 primary 上。命中策略后，OpenViking 会把映射记录到内部元数据中。用户执行 `ls()`、`stat()`、`read()` 时仍能看到正常的文件系统视图。 - -## Exclude - -Exclude 表示“某个 backup 不接收匹配的文件”。 - -常见用途： - -- 内存或缓存 backend 不保存大文件。 -- 某个 backup 只保存文本类资源。 -- 某个低成本 backend 排除临时或超大文件。 - -Exclude 策略配置在 backup 上，只影响该 backup 是否接收写入。 - -## 内部元数据 - -主备存储使用两个内部元数据文件： - -| 文件 | 作用 | -| --- | --- | -| `.redirect.json` | 记录 redirect 文件对应的目标 backend | -| `.sync_log.json` | 记录每个文件的同步版本和 backup 确认进度 | - -这些文件对普通用户不可见，不会出现在常规列表结果中，也不应通过公开 API 直接读写。 - -如果 primary 开启静态数据加密，这些内部元数据也会跟随 primary 加密策略写入。 - -## 加密关系 - -主备存储不会改变 OpenViking 的透明加密模型。 - -规则如下： - -- Python 层和公共 API 不感知加密实现。 -- primary 在全局加密开启时必须加密。 -- backup 可以独立决定是否加密。 -- 内部元数据必须走 primary 的加密入口。 - -这意味着启用主备存储后，调用方式仍然不变；只需要通过配置决定每个 backend 的加密策略。 - -## 与 OVPack 的关系 - -主备存储只负责“启用之后的新写入”。它不会自动同步启用之前已经存在于 primary 中的历史文件。 - -如果要迁移存量数据，推荐流程是： - -1. 暂停写入或安排可核对的增量补齐窗口，再使用 OVPack 或其他受控方式迁移存量。 -2. 校验目标 backend 数据。 -3. 启用主备存储配置。 -4. 补齐迁移期间的变更并验证同步状态后恢复写入；后续新增和修改由主备存储复制。 - -## 限制 - -- 异步模式下 backup 可能落后；读取 backup 时不能假设已经看到最新写入。 -- 启用主备存储前的历史文件需要单独迁移或回填。 -- redirect 文件依赖内部元数据恢复目录视图。 -- 多进程同时写同一 primary 时，需要未来的分布式元数据锁能力。 -- 热点目录会频繁更新内部元数据，可能带来额外写放大。 +V2 不提供同步状态或手工重试 API 和 CLI 命令。请使用多写 Prometheus 指标 +观察队列深度、worker 失败、backup lag、协议版本和协议状态。 ## 相关文档 - [存储架构](./05-storage.md) +- [指标](./12-metrics.md) - [配置指南](../guides/01-configuration.md) -- [主备存储指南](../guides/13-multi-write-storage.md) +- [多写存储指南](../guides/13-multi-write-storage.md) - [OVPack 导入导出](../guides/09-ovpack.md) diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 12dd465be3..66df58985b 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1290,8 +1290,7 @@ Glob 引擎配置，用于路径模式匹配。这些设置为服务端配置， |------|------|------|--------| | `backend` | str | `"local"`、`"s3"` 或 `"memory"` | `"local"` | | `timeout` | float | 请求超时时间（秒） | `10.0` | -| `backups` | object | 主备存储配置。配置后顶层 `backend` 作为 primary，`backups.items[]` 作为 backup | `null` | -| `redirects` | array | 主备存储的文件重定向策略。命中后文件写入指定 backup，而不是 primary | `[]` | +| `backups` | object | 多写存储配置。配置后顶层 `backend` 作为 primary，`backups.items[]` 作为 backup | `null` | | `queuefs` | object | QueueFS 配置。控制 `/queue` 的命名空间模式、后端和运行时参数 | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` | | `queue_db_path` | str（可选）| 旧版兼容字段，用于覆盖 QueueFS 的 sqlite 数据库文件路径。已被 `storage.agfs.queuefs.db_path` 取代。未设置时默认为 `{storage.workspace}/_system/queue/queue.db`。适用于 workspace 卷不支持 sqlite 的场景（例如某些网络文件系统） | `null` | | `s3` | object | S3 后端配置（`backend=s3` 时使用） | - | @@ -1306,7 +1305,7 @@ RAGFS 默认使用 Rust binding 模式，通过 Rust 实现直接访问文件系 ##### 主备存储配置 -`storage.agfs.backups` 用于启用主备存储。未配置时，OpenViking 保持单 backend 模式。 +`storage.agfs.backups` 启用异步 V2 多写。顶层 backend 始终是唯一直接读取来源。 ```json { @@ -1314,26 +1313,16 @@ RAGFS 默认使用 Rust binding 模式，通过 Rust 实现直接访问文件系 "workspace": "./data", "agfs": { "backend": "local", - "redirects": [ - { - "type": "FileExtensionPolicy", - "extensions": ["(pdf|ppt|zip)"], - "target": ["s3-backup"] - } - ], "backups": { - "sync_type": "async", + "initial_partitions": 16, + "checkpoint_interval_secs": 86400, + "provider": "filesystem", "items": [ { - "name": "s3-backup", - "backend": "s3", - "s3": { - "bucket": "openviking-backup", - "region": "cn-beijing", - "endpoint": "https://tos-s3-cn-beijing.volces.com", - "access_key": "your-ak", - "secret_key": "your-sk", - "prefix": "multi-write" + "name": "local-backup", + "backend": "local", + "params": { + "workspace": "./data/backup" } } ] @@ -1347,36 +1336,13 @@ RAGFS 默认使用 Rust binding 模式，通过 Rust 实现直接访问文件系 | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| -| `sync_type` | str | 主备同步模式，支持 `"async"` 或 `"sync"` | `"async"` | -| `write_ack_count` | int | `sync` 模式下返回前需要的 backup 确认数 | 全部 backup | -| `write_ack_timeout_ms` | int | `sync` 模式下等待 backup 确认的超时时间，单位毫秒 | `null` | -| `write_concurrency` | int | 异步 backup 写入并发上限 | `null` | -| `items` | array | backup backend 列表，每个 item 复用普通 backend 配置并增加 `name`、`operations`、`excludes`、`encryption` 等字段 | `[]` | - -`redirects` 常用字段： - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `type` | str | 策略类型，支持 `"FileExtensionPolicy"` 或 `"FileOverSizePolicy"` | 必填 | -| `extensions` | array | `FileExtensionPolicy` 使用的扩展名正则列表，例如 `["(pdf\|ppt)"]` | `[]` | -| `max_size_mb` | int | `FileOverSizePolicy` 使用的文件大小阈值，单位 MB | `null` | -| `target` | array | 命中策略后写入的 backup `name` 列表 | 必填 | - -按文件大小重定向示例： - -```json -{ - "type": "FileOverSizePolicy", - "max_size_mb": 100, - "target": ["s3-backup"] -} -``` - -注意： +| `initial_partitions` | int | account 首次初始化的 metadata partition 数；修改不影响已有 account | `16` | +| `checkpoint_interval_secs` | int | Checkpoint 扫描间隔，最小值为 `60` | `86400` | +| `provider` | str | Metadata provider：`filesystem` 或复用顶层 CacheRuntime 的 `cache` | `"filesystem"` | +| `items` | array | backup 列表，包含 `name`、`backend`、`params` 和可选 `encryption` | `[]` | -- `redirects` 配置在顶层 `storage.agfs`，表示 primary 的重定向策略。 -- `target` 必须引用 `backups.items[]` 中已经定义的 backup `name`。 -- 命中 redirect 的文件仍会通过普通文件系统 API 呈现为可读、可列举的文件。 +Python 配置层仍接受旧 sync、retry、operations、excludes 和 redirects +字段，输出 warning 后会在调用 Rust 前删除，字段不生效。 更多配置示例见 [主备存储指南](./13-multi-write-storage.md)。 @@ -1444,9 +1410,9 @@ RAGFS 默认使用 Rust binding 模式，通过 Rust 实现直接访问文件系 } ``` -标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`，或 PathLock 选择 `provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`，不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`，也不连接 Provider。 +标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`、PathLock 选择 `provider=cache`，或多写选择 `backups.provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`，不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`，也不连接 Provider。 -这是一次配置破坏性变更：`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"` 和 `storage.agfs.queuefs.redis` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`，业务模块改为 `cachefs.backend="cache"` 或 `queuefs.backend="cache"`；Redis 的 `singleton` 改为 `standalone`，`tls_enabled` 改为使用 `rediss://` endpoint。 +这是一次配置破坏性变更：`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"`、`storage.agfs.queuefs.redis` 和 `storage.agfs.backups.provider="redis"` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`，业务模块改为 `cachefs.backend="cache"`、`queuefs.backend="cache"` 或 `backups.provider="cache"`；Redis 的 `singleton` 改为 `standalone`，`tls_enabled` 改为使用 `rediss://` endpoint。 ##### QueueFS 配置 diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.md index 674571b085..4c1efffc27 100644 --- a/docs/zh/guides/08-encryption.md +++ b/docs/zh/guides/08-encryption.md @@ -21,7 +21,7 @@ OpenViking 可以按账户加密存储中新写入的文件。加解密由存储 - 全局 `encryption.enabled=true` 时，primary backend 必须加密。 - backup backend 可以通过自己的 `encryption.enabled` 控制是否加密。 -- `.redirect.json` 和 `.sync_log.json` 等主备存储内部元数据跟随 primary 加密策略。 +- V2 多写 metadata 存储在 raw primary，不经过 `EncryptionWrappedFS`。 - OpenViking 不提供也不需要公开的加解密 API 来操作这些内部文件。 更多主备存储配置见 [主备存储指南](./13-multi-write-storage.md)。 diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.md index a611267dbf..9533748a88 100644 --- a/docs/zh/guides/09-ovpack.md +++ b/docs/zh/guides/09-ovpack.md @@ -33,22 +33,12 @@ Session 通过 user 命名空间一起迁移，路径为 ## 与主备存储配合 -主备存储只复制启用之后的新写入，不会自动同步启用之前已经存在的历史文件。如需同时为 primary 和副本写入存量，应在**空目标环境恢复之前**启用主备存储： +V2 启动会把当前 V1 primary 文件导入分区 metadata，并删除 V1 sidecar。 +导入不会删除 backup 中多余的旧文件。首次 V2 启动前，应停服所有实例并创建 +完整备份。 -1. 暂停业务写入，按源 account 分别导出或备份。 -2. 配置目标 primary、backup backend 及其写策略，启动目标服务并创建恢复身份。 -3. 通过该服务恢复或导入，使存量内容经过已配置的主备复制分发。 -4. 对每个恢复的 scope 检查同步状态，并在切流前验证各副本中的文件。恢复返回时，异步复制仍可能未完成。 -5. 验证内容和索引完整性后再恢复业务写入；验证完成前保留源数据和备份。 - -例如，使用目标 account 的 admin key 检查： - -```bash -ov system backend sync-status viking://resources -ov system backend sync-status viking://user -``` - -先恢复再启用 backups 只会复制后续写入，不能把历史内容补到副本。 +导入开始后不支持原地回滚。要恢复 V1，必须恢复导入前的完整备份。启动后使用 +普通文件 API 验证 primary 内容，并通过 Prometheus 指标观察 backup lag。 更多说明见 [主备存储指南](./13-multi-write-storage.md)。 diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.md index 14d5e2dc1f..3211082dbc 100644 --- a/docs/zh/guides/13-multi-write-storage.md +++ b/docs/zh/guides/13-multi-write-storage.md @@ -1,20 +1,10 @@ -# 主备存储指南 +# 多写存储指南 -本指南介绍如何配置 OpenViking 的主备存储能力。主备存储允许一个 primary 后端同时复制写入多个 backup 后端，用于保存副本（包括跨地域副本）、配置读路由和存储迁移。primary 故障时不会自动提升 backup。 - -主备存储能力位于 RAGFS 内部。OpenViking 的 Python SDK、HTTP API 和 CLI 使用方式保持不变。 - -## 前置条件 - -- 已有可用的 `ov.conf`。 -- 已确认 primary backend 可以正常读写。 -- 如果要接入 S3 兼容存储，已准备好 bucket、endpoint 和访问凭据。 -- 如需迁移已有数据，请遵循 [OVPack 主备存储迁移流程](./09-ovpack.md#与主备存储配合)；仅启用 backups 不会复制历史文件。 +V2 多写在 primary 成功后返回，metadata 和 backup 在后台异步更新。primary +是唯一直接读取来源。 ## 最小配置 -将下列片段合并到现有模型和服务端配置。下面示例使用本地目录作为 primary，并把写入复制到另一个本地目录。 - ```json { "storage": { @@ -22,13 +12,15 @@ "agfs": { "backend": "local", "backups": { - "sync_type": "async", + "initial_partitions": 16, + "checkpoint_interval_secs": 86400, + "provider": "filesystem", "items": [ { "name": "local-backup", "backend": "local", - "local": { - "workspace": "./data/backup" + "params": { + "workspace": "./data/backup" } } ] @@ -38,385 +30,55 @@ } ``` -说明： - -- 顶层 `backend` 是 primary。 -- `backups.items[]` 是 backup 列表。 -- `name` 是 backup 的稳定身份，后续同步元数据会引用它。 -- `backend = "local"` 的 backup 使用 `local.workspace` 指定本地目录。 -- `sync_type` 不配置时默认按异步模式理解。 - -## 多 Backup 配置 +V2 的三个顶层配置是： -可以配置多个 backup。下面示例同时写入本地副本和 S3 兼容对象存储。 - -```json -{ - "storage": { - "workspace": "./data", - "agfs": { - "backend": "local", - "backups": { - "sync_type": "async", - "items": [ - { - "name": "local-az2", - "backend": "local", - "local": { - "workspace": "./data/local-az2" - } - }, - { - "name": "object-store", - "backend": "s3", - "s3": { - "bucket": "openviking-backup", - "region": "us-east-1", - "endpoint": "https://s3.example.com", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "prefix": "openviking", - "directory_marker_mode": "none" - } - } - ] - } - } - } -} -``` - -建议： - -- `name` 不要使用会频繁变化的机器名或临时编号。 -- backup 的底层路径或 bucket 应避免与 primary 指向同一物理位置。 -- 修改 backup `name` 会影响历史同步元数据的识别，生产环境应谨慎变更。 - -### S3 兼容存储注意事项 - -按实际 endpoint 和对象存储配置选择以下 `s3` 字段： - -| 字段 | 是否必填 | 说明 | +| 字段 | 默认值 | 含义 | | --- | --- | --- | -| `use_path_style` | 可选，默认 `true` | 使用路径风格 URL（`http://host/bucket/key`），按 endpoint 支持的方式选择。 | -| `directory_marker_mode` | backup 项必填 | `none` 不创建标记，`empty` 创建零字节标记，`nonempty` 创建带内容的标记。每个 S3 backup 项都要显式设置；省略时启动会报 `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`。 | -| `use_ssl` | 可选 | HTTP 端点（如 `http://localhost:9000`）需要设置为 `false`。 | - -**S3 兼容存储最小示例（RustFS/MinIO）：** - -```json -{ - "name": "s3-backup", - "backend": "s3", - "s3": { - "bucket": "my-bucket", - "endpoint": "http://localhost:9000", - "access_key": "your-access-key", - "secret_key": "your-secret-key", - "prefix": "openviking", - "use_ssl": false, - "use_path_style": true, - "directory_marker_mode": "none" - } -} -``` - -上面的示例使用 `none`，适用于不使用目录标记的 S3 兼容服务（RustFS、MinIO、Ceph 等）。backup 项的这个字段没有默认值；顶层 `storage` 的 S3 配置默认使用 `empty`。 - -### Docker 网络配置 - -在 Docker 中运行 OpenViking 并配置同主机的 S3 备份时，需要注意： - -- **Linux Docker**：使用 `--network host` 或宿主机局域网 IP。Docker bridge 网络可通过网关 IP（如 `172.17.0.1:9000`）访问宿主机局域网。 -- **macOS/Windows Docker Desktop**：从容器访问宿主机时可使用 `host.docker.internal`。Docker Desktop 4.34 及之后的版本也提供可选的 host networking，启用方式见 [Docker 官方说明](https://docs.docker.com/engine/network/drivers/host/)。 - -如果启用 S3 备份后服务静默崩溃，请优先排查 Docker 网络。RAGFS Rust binding 在容器内无法访问 S3 端点时会报 `dispatch failure` 错误。 - -## 同步模式选择 - -### 异步模式 - -异步模式适合大多数场景。 - -```json -{ - "backups": { - "sync_type": "async", - "items": [] - } -} -``` - -特点： - -- primary 写入成功后立即返回。 -- backup 写入在后台执行。 -- 写入延迟低。 -- 后端不可用期间，落后时间没有固定上限，需观察同步和恢复状态。 - -适合： - -- 写入吞吐优先。 -- backup 主要用于灾备。 -- 可以接受最终一致性。 - -### 同步模式 - -同步模式会等待 backup 确认。 - -```json -{ - "backups": { - "sync_type": "sync", - "write_ack_count": 1, - "write_ack_timeout_ms": 5000, - "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}] - } -} -``` - -参数说明： - -| 参数 | 说明 | -| --- | --- | -| `write_ack_count` | 写入返回前至少需要多少个 backup 确认 | -| `write_ack_timeout_ms` | 等待 backup 确认的超时时间，单位毫秒 | - -特点： - -- 写入确认更强。 -- 写入延迟受 backup 影响。 -- 未确认的 backup 会继续由后台重试修复。 -- primary 已写成功但 backup 未达确认数时，客户端可能收到错误；此时 primary 中可能已经存在数据。 - -适合： - -- 希望尽量减少 primary 与 backup 的确认窗口。 -- backup 延迟可控。 -- 调用方能接受同步写入带来的额外延迟。 - -## 配置读加速 - -backup 默认不参与读取。要让 backup 服务读取，需要显式配置 `operations`。 - -```json -{ - "name": "cache-backend", - "backend": "memfs", - "operations": [ - {"operation": "write", "priority": 0}, - { - "operation": "read", - "priority": 10 - } - ] -} -``` - -读取优先级规则： - -- `priority` 越小越优先。 -- 只有声明 `read` 的 backup 才参与读取。 -- primary 始终作为最终兜底。 -- 冷备 backup 不建议配置读能力。 - -如果一个 backup 只配置了 `read`，没有配置 `write`，它不会接收普通复制写入。只有在你明确知道该 backend 的数据来源时，才应使用这种配置。 +| `initial_partitions` | `16` | 为 account 创建的 metadata partition 数量 | +| `checkpoint_interval_secs` | `86400` | Checkpoint 扫描间隔，最小值为 `60` | +| `provider` | `"filesystem"` | Metadata segment provider：`filesystem` 或 `cache` | -## Redirect 配置 +每个 `items[]` 需要 `name`、`backend` 和 backend 专用 `params`。 +`encryption` 可选。backup 名称必须唯一，且不能为 `primary`。 -Redirect 用于把匹配的文件写入指定 backup，而不是写入 primary。 +`provider` 为 `cache` 时，多写复用顶层 CacheRuntime。实际实现由顶层 +`cache.provider` 选择；多写没有独立 Cache Provider 连接配置。 -按扩展名重定向： +Python 配置层仍接受 V1 的 sync、ack、retry、operations、excludes 和 +redirects 字段，输出 warning 后会在调用 Rust 前删除，字段不生效。 -```json -{ - "storage": { - "agfs": { - "backend": "local", - "redirects": [ - { - "type": "FileExtensionPolicy", - "extensions": ["(pdf|ppt|zip)"], - "target": ["object-store"] - } - ], - "backups": { - "items": [ - { - "name": "object-store", - "backend": "s3", - "s3": { - "bucket": "openviking-large-files", - "endpoint": "https://s3.example.com" - } - } - ] - } - } - } -} -``` - -按大小重定向： - -```json -{ - "type": "FileOverSizePolicy", - "max_size_mb": 100, - "target": ["object-store"] -} -``` - -注意： - -- `target` 必须引用已有 backup 的 `name`。 -- redirect 文件仍会通过普通 API 呈现为可读、可列举、可查询状态。 -- redirect 映射保存在 primary 的内部元数据中。 - -## Exclude 配置 - -Exclude 用于让某个 backup 跳过匹配文件。 - -```json -{ - "name": "cache-backend", - "backend": "memfs", - "excludes": [ - { - "type": "FileOverSizePolicy", - "max_size_mb": 50 - }, - { - "type": "FileExtensionPolicy", - "extensions": ["(mp4|zip)"] - } - ] -} -``` - -常见用法： - -- 缓存 backend 排除大文件。 -- 低成本备份排除无需保存的文件类型。 -- 某个 backup 只保存文本或配置类资源。 - -如果 redirect 的目标 backup 同时 exclude 了该文件，说明配置互相冲突。请优先修正配置，不要依赖系统自动猜测其他目标。 - -## 加密配置 - -主备存储复用 OpenViking 的透明静态加密能力。 - -全局加密开启示例： - -```json -{ - "encryption": { - "enabled": true, - "provider": "local", - "local": { - "key_file": "~/.openviking/master.key" - } - }, - "storage": { - "workspace": "./data", - "agfs": { - "backend": "local", - "backups": { - "items": [ - { - "name": "plain-cache", - "backend": "memfs", - "encryption": { - "enabled": false - } - }, - { - "name": "encrypted-backup", - "backend": "local", - "local": { - "workspace": "./data/encrypted-backup" - }, - "encryption": { - "enabled": true - } - } - ] - } - } - } -} -``` - -规则： - -- 全局 `encryption.enabled=true` 时，primary 必须加密。 -- backup 可以通过 `encryption.enabled` 单独控制是否加密。 -- Python SDK、HTTP API 和 CLI 不需要处理加解密。 -- `.redirect.json` 和 `.sync_log.json` 等内部元数据会跟随 primary 加密策略。 +## 运行行为 -## 存量数据迁移 - -主备存储只复制启用之后的新写入，不会自动复制历史文件。 - -使用 OVPack 迁移时，请遵循 [与主备存储配合](./09-ovpack.md#与主备存储配合)：先在干净目标配置主备存储，再通过该服务恢复，验证各副本后恢复业务写入。目标 account 初始化和恢复冲突处理也在该流程中说明。 - -## 验证配置 - -启动前建议运行： - -```bash -openviking-server doctor -``` - -启动后可以用普通文件 API 验证： - -```bash -ov write viking://resources/multiwrite-check.txt \ - --content "multi-write check" - -ov read viking://resources/multiwrite-check.txt -``` - -如果使用本地 backup，可以直接检查 backup 目录中是否出现对应文件。还应运行 `ov system backend sync-status viking://resources`，逐个核对预期副本。普通读取成功可能只读到了 primary，不能证明复制完成。 - -## 常见问题 - -### 为什么 backup 没有参与读取？ - -backup 默认只参与写入，不参与读取。需要在 backup 上显式配置： - -```json -{ - "operations": [ - {"operation": "write", "priority": 0}, - { - "operation": "read", - "priority": 10 - } - ] -} -``` +- primary 是唯一直接读取来源和业务真相。 +- primary 成功后立即返回，不等待 metadata 或 backup。 +- 多写 metadata 只存储在 primary。 +- backup 在后台最终收敛。 +- 不提供同步状态或手工重试 API 和 CLI 命令。 -### 为什么启用主备存储后历史文件没有出现在 backup？ +进程可能在 primary 成功和事件持久化之间崩溃，导致一次复制事件丢失。此时 +需要从 primary 重建受影响的 backup。 -主备存储只处理启用后的新写入。历史文件请按 [OVPack 迁移流程](./09-ovpack.md#与主备存储配合) 处理；启用 backups 不会自动补齐。 +## Partition 数量 -### 异步模式下能否保证立即读到 backup 的最新数据？ +`initial_partitions` 仅在 account 的 partition manifest 首次创建时生效。 +已有 account 在配置变化后继续使用持久化的 partition 数量和 routes。 +本期不支持扩容或缩容。 -不能。后台重试需要后端恢复可用才能追上进度。backup 返回旧数据但读取成功时，不会自动回退 primary；需要最新数据时，应让读路由使用 primary。 +## 首次启用与回滚 -### 内部元数据文件会出现在用户列表里吗？ +`.multiwrite.json` 不存在时，mount 先启用前台 V2 事件提交，再在后台导入 +primary 当前文件。导入不会删除 backup 中多余的旧文件。 -不会。`.redirect.json` 和 `.sync_log.json` 是内部文件，会被普通目录列表隐藏。 +旧多写协议和旧迁移字段直接拒绝。启用多写前必须创建完整备份；回滚二进制 +前必须先恢复该备份。 -### sync 模式返回失败是否表示 primary 一定没写入？ +## 验证 -不是。primary 写成功但 backup 未达到确认数时，客户端可能收到失败。此时 primary 数据可能已经存在，落后的 backup 会由后台重试修复。 +使用普通文件 API 验证 primary 读写。使用[指标](../concepts/12-metrics.md) +监控队列深度、worker 失败、backup lag、协议版本和协议状态。 ## 相关文档 -- [主备存储](../concepts/14-multi-write-storage.md) -- [存储架构](../concepts/05-storage.md) +- [多写存储](../concepts/14-multi-write-storage.md) - [配置指南](./01-configuration.md) -- [加密指南](./08-encryption.md) - [OVPack 导入导出](./09-ovpack.md) diff --git a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json b/examples/claude-code-memory-plugin/.claude-plugin/plugin.json index 627d750e5c..c839bd3b22 100644 --- a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json +++ b/examples/claude-code-memory-plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "openviking-memory", - "version": "0.7.10", + "version": "0.7.11", "description": "Long-term semantic memory for Claude Code, powered by OpenViking. Auto-recall relevant memories at session start and capture important information during conversations.", "author": { "name": "OpenViking", @@ -17,4 +17,4 @@ ], "mcpServers": "./.mcp.json", "types": "./mods/usage/types.d.ts" -} +} \ No newline at end of file diff --git a/examples/claude-code-memory-plugin/package.json b/examples/claude-code-memory-plugin/package.json index 71eb2f10f1..52205300ac 100644 --- a/examples/claude-code-memory-plugin/package.json +++ b/examples/claude-code-memory-plugin/package.json @@ -1,6 +1,6 @@ { "name": "claude-code-openviking-memory", - "version": "0.7.10", + "version": "0.7.11", "description": "OpenViking memory plugin for Claude Code — semantic long-term memory", "type": "module", "scripts": { diff --git a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md index 684d8172a8..04c78ecc9f 100644 --- a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md +++ b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md @@ -159,7 +159,7 @@ port is bound only after initialization finishes, so "connection refused" during startup is normal; `/health` 200 says nothing about embedding or VLM. `GET /ready` (no auth, 200 or 503): -`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`. +`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`. `ok`, `not_configured` and `not_supported` count as healthy; `embedding` is a real embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}` while booting; 404 on servers that predate the endpoint. The official docker diff --git a/examples/codex-memory-plugin/.codex-plugin/plugin.json b/examples/codex-memory-plugin/.codex-plugin/plugin.json index 2a3f34ac25..4870aa30ad 100644 --- a/examples/codex-memory-plugin/.codex-plugin/plugin.json +++ b/examples/codex-memory-plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "openviking-memory", - "version": "0.11.9", + "version": "0.11.10", "description": "Long-term semantic memory for Codex, powered by OpenViking. Recall on UserPromptSubmit, incremental add_message on Stop (per turn), commit on SessionEnd (Codex >= 0.145) and on PreCompact, with an idle/ended sweep on SessionStart as the fallback for signals, crashes and older Codex builds. Includes a local stdio MCP proxy that reads OpenViking credentials from env/ovcli.conf.", "author": { "name": "OpenViking", diff --git a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md index 63b6a73c89..6e36ee3001 100644 --- a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md +++ b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md @@ -152,7 +152,7 @@ port is bound only after initialization finishes, so "connection refused" during startup is normal; `/health` 200 says nothing about embedding or VLM. `GET /ready` (no auth, 200 or 503): -`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`. +`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`. `ok`, `not_configured` and `not_supported` count as healthy; `embedding` is a real embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}` while booting; 404 on servers that predate the endpoint. The official docker diff --git a/examples/memory-plugin-shared/doctor-core.test.mjs b/examples/memory-plugin-shared/doctor-core.test.mjs index 8ff2b844ce..86d8bf55b4 100644 --- a/examples/memory-plugin-shared/doctor-core.test.mjs +++ b/examples/memory-plugin-shared/doctor-core.test.mjs @@ -237,7 +237,7 @@ test("lintServerConf covers every harness, not just the two with a legacy ov.con }); test("assessReady interprets the readiness checks", () => { - const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok", multiwrite_sync: "not_supported" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } }; + const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } }; let report = createReport(); assert.equal(assessReady(report, ok).ready, true); assert.equal(report.exitCode(), 0); diff --git a/examples/ov.conf.example b/examples/ov.conf.example index 5b81cb297e..8650d0b0c0 100644 --- a/examples/ov.conf.example +++ b/examples/ov.conf.example @@ -77,6 +77,17 @@ "agfs": { "backend": "local", "timeout": 10, + "backups_example": { + "_comment": "Rename this key to backups to enable V2 multi-write.", + "initial_partitions": 16, + "checkpoint_interval_secs": 86400, + "provider": "cache", + "items": [{ + "name": "local-backup", + "backend": "local", + "params": {"workspace": "./data/backup"} + }] + }, "cachefs": { "backend": "local", "namespace": "openviking", diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py index ebfa1d5633..4c6657e8ca 100644 --- a/openviking/pyagfs/async_client.py +++ b/openviking/pyagfs/async_client.py @@ -349,18 +349,6 @@ async def glob_directory( ctx=_fs_ctx_or_default(path, fs_ctx), ) - async def system_sync_status( - self, path: str, *, fs_ctx: Dict[str, str] | None = None - ) -> Dict[str, Any]: - """Return multi-write sync status for a file or directory path.""" - return await self.run("system_sync_status", path, ctx=_fs_ctx_or_default(path, fs_ctx)) - - async def system_sync_retry( - self, path: str, *, fs_ctx: Dict[str, str] | None = None - ) -> Dict[str, Any]: - """Retry pending multi-write sync work for a file or directory path.""" - return await self.run("system_sync_retry", path, ctx=_fs_ctx_or_default(path, fs_ctx)) - # -- pathlock async wrappers ------------------------------------------------ async def pathlock_acquire_exact( diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.py index 020151cb8c..5c2a484b2a 100644 --- a/openviking/pyagfs/protocols.py +++ b/openviking/pyagfs/protocols.py @@ -118,9 +118,3 @@ def glob_directory( continuation_token: str | None = None, ) -> Dict[str, Any]: """Return one page of flat glob matches for the given AGFS directory.""" - - def system_sync_status(self, path: str) -> Dict[str, Any]: - """Return multi-write sync status for a file or directory path.""" - - def system_sync_retry(self, path: str) -> Dict[str, Any]: - """Retry pending multi-write sync work for a file or directory path.""" diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.py index 686444728d..75d44bb926 100644 --- a/openviking/server/auth/plugins/api_key.py +++ b/openviking/server/auth/plugins/api_key.py @@ -24,8 +24,6 @@ "/api/v1/observer", "/api/v1/console", "/api/v1/tasks", - "/api/v1/system/backend", - "/api/v1/system/sync", ) diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.py index 0ef245f79a..78b62f4d61 100644 --- a/openviking/server/routers/system.py +++ b/openviking/server/routers/system.py @@ -38,19 +38,13 @@ def _is_ready_check_ok(value) -> bool: async def _probe_agfs_readiness() -> dict[str, object]: - """Return structured AGFS readiness, including multi-write sync health when available.""" + """Return structured AGFS filesystem readiness.""" viking_fs = get_viking_fs() checks: dict[str, object] = {} await viking_fs.ls("viking://", ctx=None) checks["filesystem"] = "ok" - try: - await viking_fs.system_sync_status("viking://", ctx=None) - checks["multiwrite_sync"] = "ok" - except (AGFSInvalidOperationError, AGFSNotSupportedError): - checks["multiwrite_sync"] = "not_supported" - return {"status": "ok", "checks": checks} @@ -126,7 +120,7 @@ async def readiness_check(request: Request): checks = {} - # 1. AGFS: probe filesystem access and multi-write sync health + # 1. AGFS: probe filesystem access try: checks["agfs"] = await _probe_agfs_readiness() except Exception as e: @@ -220,12 +214,6 @@ class ConsistencyRequest(BaseModel): uri: str -class BackendSyncRequest(BaseModel): - """Request model for backend sync status and retry operations.""" - - uri: str - - @router.post("/api/v1/system/wait", tags=["system"]) async def wait_processed( request: WaitRequest, @@ -250,51 +238,3 @@ async def check_consistency( ctx=ctx, ) return Response(status="ok", result=result) - - -@router.post("/api/v1/system/backend/sync-status", tags=["system"]) -async def backend_sync_status( - request: BackendSyncRequest, - ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN), -): - """Return multi-write backend sync status for a Viking URI subtree.""" - service = get_service() - uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx) - result = await service.fs.system_sync_status(uri, ctx=ctx) - return Response(status="ok", result=result) - - -@router.post("/api/v1/system/backend/sync-retry", tags=["system"]) -async def backend_sync_retry( - request: BackendSyncRequest, - ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN), -): - """Retry pending multi-write backend sync work for a Viking URI subtree.""" - service = get_service() - uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx) - result = await service.fs.system_sync_retry(uri, ctx=ctx) - return Response(status="ok", result=result) - - -@router.get("/api/v1/system/sync/{sync_path:path}", tags=["system"]) -async def admin_sync_status( - sync_path: str, - ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN), -): - """Return multi-write backend sync status for one URI subtree through the admin API.""" - service = get_service() - uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx) - result = await service.fs.system_sync_status(uri, ctx=ctx) - return Response(status="ok", result=result) - - -@router.post("/api/v1/system/sync/{sync_path:path}/retry", tags=["system"]) -async def admin_sync_retry( - sync_path: str, - ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN), -): - """Retry pending multi-write backend sync work for one URI subtree through the admin API.""" - service = get_service() - uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx) - result = await service.fs.system_sync_retry(uri, ctx=ctx) - return Response(status="ok", result=result) diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py index 163e75e663..ee32386a7e 100644 --- a/openviking/service/fs_service.py +++ b/openviking/service/fs_service.py @@ -1062,16 +1062,6 @@ async def ensure_write_access(self, uri: str, ctx: RequestContext) -> None: viking_fs = self._ensure_initialized() await viking_fs._ensure_access(uri, ctx, action=AclAction.WRITE) - async def system_sync_status(self, uri: str, ctx: RequestContext) -> Dict[str, Any]: - """Return multi-write sync status for one Viking URI subtree.""" - viking_fs = self._ensure_initialized() - return await viking_fs.system_sync_status(uri, ctx=ctx) - - async def system_sync_retry(self, uri: str, ctx: RequestContext) -> Dict[str, Any]: - """Retry multi-write sync work for one Viking URI subtree.""" - viking_fs = self._ensure_initialized() - return await viking_fs.system_sync_retry(uri, ctx=ctx) - async def read(self, uri: str, ctx: RequestContext, offset: int = 0, limit: int = -1) -> str: """Read file content. Accepts a Viking URI or a 32-char hex vector record id.""" viking_fs = self._ensure_initialized() diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.py index 92dc4df79b..c5f561947c 100644 --- a/openviking/storage/viking_fs/_snapshot.py +++ b/openviking/storage/viking_fs/_snapshot.py @@ -121,30 +121,6 @@ async def _ensure_restore_plan_access( def _restore_reindex_context(ctx: RequestContext) -> RequestContext: return replace(ctx, bypass_acl=True) - async def system_sync_status( - self, uri: str, ctx: Optional[RequestContext] = None - ) -> Dict[str, Any]: - """Return multi-write sync status for one Viking URI subtree.""" - await self._ensure_access(uri, ctx) - real_ctx = self._ctx_or_default(ctx) - path = self._uri_to_path(uri, ctx=ctx) - return await self._async_agfs.system_sync_status( - path, - fs_ctx={"account_id": real_ctx.account_id}, - ) - - async def system_sync_retry( - self, uri: str, ctx: Optional[RequestContext] = None - ) -> Dict[str, Any]: - """Retry multi-write sync for one Viking URI subtree.""" - await self._ensure_access(uri, ctx, action=AclAction.WRITE) - real_ctx = self._ctx_or_default(ctx) - path = self._uri_to_path(uri, ctx=ctx) - return await self._async_agfs.system_sync_retry( - path, - fs_ctx={"account_id": real_ctx.account_id}, - ) - async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str: """Return the account-level .ovgitignore content, or an empty string if absent. diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py index 6f6fc7b836..6acc259e15 100644 --- a/openviking/utils/agfs_utils.py +++ b/openviking/utils/agfs_utils.py @@ -41,7 +41,13 @@ def to_binding_dict(self) -> Dict[str, Any]: cachefs_backend = getattr(cachefs, "backend", "local") pathlock = self.agfs.pathlock.model_dump(mode="json", exclude_none=True) pathlock_uses_cache = pathlock.get("provider") == "cache" - uses_runtime = cachefs_backend == "cache" or queue_backend == "cache" or pathlock_uses_cache + backups = getattr(self.agfs, "backups", None) + uses_runtime = ( + cachefs_backend == "cache" + or queue_backend == "cache" + or pathlock_uses_cache + or _get_config_value(backups, "provider", "filesystem") == "cache" + ) if ( pathlock_uses_cache and self.cache is not None @@ -60,8 +66,8 @@ def to_binding_dict(self) -> Dict[str, Any]: else: if uses_runtime: raise ValueError( - "top-level cache config is required when CacheFS, QueueFS, or cache-backed PathLock " - "uses the shared Runtime" + "top-level cache config is required when CacheFS, QueueFS, cache-backed PathLock, " + "or cache-backed multi-write uses the shared Runtime" ) cache_config = _disabled_cache_config(cachefs) binding_config: Dict[str, Any] = { @@ -301,10 +307,8 @@ def _generate_plugin_config( # Check for multi-write configuration backups_config = getattr(agfs_config, "backups", None) redirects_config = getattr(agfs_config, "redirects", None) - if redirects_config is not None and backups_config is None: - raise ValueError( - "redirects requires backups; single-backend mode does not support redirects" - ) + if redirects_config is not None: + logger.warning("storage.agfs.redirects is deprecated and ignored") # Build primary backend plugin config primary_plugin_config: Dict[str, Any] = {} @@ -326,12 +330,6 @@ def _generate_plugin_config( mount_config["server_encryption_enabled"] = server_encryption_enabled mount_config["primary_encryption_enabled"] = server_encryption_enabled - # Serialize redirect policies - if redirects_config is not None: - mount_config["primary_redirects"] = [ - _serialize_redirect_policy(p) for p in redirects_config - ] - # Determine the plugin type name for the primary backend if backend == "local": plugin_name = "localfs" @@ -370,6 +368,9 @@ def _get_config_value(config: Any, key: str, default: Any = None) -> Any: def _get_backend_specific_params(item: Any) -> Any: """Return backend-specific nested params from one backup item using the backend name key.""" + params = _get_config_value(item, "params") + if params is not None: + return params backend_params = _get_config_value(item, "backend_params") if backend_params is not None: return backend_params @@ -482,21 +483,42 @@ def _normalize_backup_item(item: Any, data_path: Path) -> Dict[str, Any]: def _serialize_backups_config(backups_config: Any, data_path: Path) -> Dict[str, Any]: - """Serialize raw multi-write backups config with top-level passthrough and normalized items.""" + """Serialize V2 backups while warning about and stripping legacy fields.""" result = _dump_config_object(backups_config) - result.setdefault("sync_type", "async") - result["items"] = [ - _normalize_backup_item(item, data_path) - for item in _get_config_value(backups_config, "items", []) - ] + deprecated_fields = [] + for field_name in ( + "sync_type", + "write_ack_count", + "write_ack_timeout_ms", + "write_concurrency", + "retry_interval_ms", + "retry_backoff_base_ms", + "retry_max_retries_per_round", + "retry_quarantine_after_failures", + "read_probe_cache_ttl_ms", + ): + if field_name in result: + deprecated_fields.append(field_name) + result.pop(field_name) + + items = [] + for item in _get_config_value(backups_config, "items", []): + normalized = _normalize_backup_item(item, data_path) + for field_name in ("timeout", "operations", "excludes"): + if field_name in normalized: + deprecated_fields.append(f"items.{field_name}") + normalized.pop(field_name) + items.append(normalized) + result["items"] = items + + if deprecated_fields: + logger.warning( + "Deprecated backups configuration fields are ignored: %s", + ", ".join(dict.fromkeys(deprecated_fields)), + ) return result -def _serialize_redirect_policy(policy: Any) -> Dict[str, Any]: - """Serialize one raw redirect/exclude policy object to a dict.""" - return _dump_config_object(policy) - - def _build_git_config_dict(git_config: Any, storage_path: Path) -> Dict[str, Any]: """Build an in-memory git config dict consumed by the ragfs binding. diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py index a9afb07aac..fb8de031be 100644 --- a/openviking_cli/utils/config/agfs_config.py +++ b/openviking_cli/utils/config/agfs_config.py @@ -446,7 +446,7 @@ class AGFSConfig(BaseModel): default=None, description="Multi-write backups configuration. None = single backend mode." ) redirects: Optional[List[dict[str, Any]]] = Field( - default=None, description="Primary redirect policies." + default=None, description="[Deprecated] Primary redirect policies. Ignored by V2." ) @model_validator(mode="after") @@ -494,9 +494,38 @@ def validate_config(self): "db_path/queue_db_path will be ignored." ) - if self.redirects is not None and self.backups is None: - raise ValueError( - "redirects requires backups; single-backend mode does not support redirects" - ) + if self.backups is not None: + initial_partitions = self.backups.get("initial_partitions", 16) + if ( + not isinstance(initial_partitions, int) + or isinstance(initial_partitions, bool) + or not 1 <= initial_partitions <= 1024 + ): + raise ValueError("backups initial_partitions must be between 1 and 1024") + + checkpoint_interval_secs = self.backups.get("checkpoint_interval_secs", 86400) + if ( + not isinstance(checkpoint_interval_secs, int) + or isinstance(checkpoint_interval_secs, bool) + or checkpoint_interval_secs < 60 + ): + raise ValueError("backups checkpoint_interval_secs must be at least 60") + + if self.backups.get("provider", "filesystem") not in {"filesystem", "cache"}: + raise ValueError("backups provider must be 'filesystem' or 'cache'") + + names = set() + for item in self.backups.get("items", []): + if not isinstance(item, dict): + raise ValueError("backups items must be objects") + name = item.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError("backup name must not be empty") + name = name.strip() + if name == "primary": + raise ValueError("backup name 'primary' is reserved") + if name in names: + raise ValueError("backup names must be unique") + names.add(name) return self diff --git a/tests/agfs/test_async_client.py b/tests/agfs/test_async_client.py index 9c8b025d44..cb08977711 100644 --- a/tests/agfs/test_async_client.py +++ b/tests/agfs/test_async_client.py @@ -6,10 +6,17 @@ import pytest -from openviking.pyagfs import AsyncAGFSClient +from openviking.pyagfs import AsyncAGFSClient, RAGFSBindingClient from openviking.storage.viking_vector_index_backend import _AsyncVectorAdapter +def test_removed_sync_methods_are_not_exposed(): + """Removed sync methods must stay absent from Python and native clients.""" + for client_type in (AsyncAGFSClient, RAGFSBindingClient): + assert not hasattr(client_type, "system_sync_status") + assert not hasattr(client_type, "system_sync_retry") + + @pytest.mark.asyncio @pytest.mark.parametrize("backend", ["agfs", "vector"]) async def test_async_agfs_client_hides_threadpool(backend): diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py index 76a73cbb73..a632835c83 100644 --- a/tests/misc/test_config_validation.py +++ b/tests/misc/test_config_validation.py @@ -440,59 +440,62 @@ def test_generate_plugin_config_forwards_queuefs_runtime_options(): assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234 -def test_agfs_redirects_require_backups(): - """Single-backend mode must reject redirect policies during config validation.""" - with pytest.raises(ValueError, match="redirects requires backups"): - AGFSConfig( - path="/tmp/ov-test", - backend="local", - redirects=[ - { - "type": "FileExtensionPolicy", - "extensions": ["(md)"], - "target": ["s3-backup"], - } - ], - ) +def test_agfs_redirects_without_backups_are_parse_only(): + """Legacy redirects must remain constructible without enabling multi-write.""" + redirects = [ + { + "type": "FileExtensionPolicy", + "extensions": ["(md)"], + "target": ["missing-backup"], + } + ] + + config = AGFSConfig( + path="/tmp/ov-test", + backend="local", + redirects=redirects, + ) + assert config.redirects == redirects -def test_generate_plugin_config_rejects_redirects_without_backups(tmp_path): - """Runtime plugin config generation must also reject redirect-only configs.""" - config = type( - "RedirectOnlyConfig", - (), + +def test_generate_plugin_config_ignores_redirects_without_backups(tmp_path): + """Legacy redirects must not alter a single-backend mount config.""" + redirects = [ { - "backend": "local", - "s3": None, - "backups": None, - "redirects": [ - type( - "RedirectPolicy", - (), - { - "type": "FileExtensionPolicy", - "extensions": ["(md)"], - "target": ["s3-backup"], - }, - )() - ], - "queuefs": type( - "QueueConfig", - (), - { - "mode": "shared", - "backend": "sqlite", - "db_path": None, - "recover_stale_sec": 0, - "busy_timeout_ms": 5000, - }, - )(), - "queue_db_path": None, - }, - )() + "type": "FileExtensionPolicy", + "extensions": ["(md)"], + "target": ["missing-backup"], + } + ] + config = AGFSConfig(path=str(tmp_path), backend="local", redirects=redirects) + plain_config = AGFSConfig(path=str(tmp_path), backend="local") + + mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"] + plain_mount_config = _generate_plugin_config(plain_config, tmp_path)["localfs"]["config"] + + assert mount_config == plain_mount_config + + +def test_generate_plugin_config_ignores_redirects_with_backups(tmp_path): + """Legacy redirects must remain accepted without reaching Rust.""" + redirects = [ + { + "type": "FileExtensionPolicy", + "extensions": ["(md)"], + "target": ["missing-backup"], + } + ] + config = AGFSConfig( + path=str(tmp_path), + backend="local", + backups={"items": [{"name": "mem-backup", "backend": "memory"}]}, + redirects=redirects, + ) + + mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"] - with pytest.raises(ValueError, match="redirects requires backups"): - _generate_plugin_config(config, tmp_path) + assert "primary_redirects" not in mount_config def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path): @@ -517,20 +520,85 @@ def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path): assert mount_config["primary_encryption_enabled"] is True -def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path): - """Plugin config generation should normalize backup params while preserving top-level multi-write fields.""" +def test_generate_plugin_config_preserves_v2_backups_without_sync_type(tmp_path): + """V2 backup fields must pass through without injecting a V1 sync mode.""" + config = AGFSConfig( + path=str(tmp_path), + backend="local", + backups={ + "initial_partitions": 32, + "checkpoint_interval_secs": 120, + "provider": "filesystem", + "items": [{"name": "mem-backup", "backend": "memory"}], + }, + ) + + backups = _generate_plugin_config(config, tmp_path)["localfs"]["config"]["backups"] + + assert backups["initial_partitions"] == 32 + assert backups["checkpoint_interval_secs"] == 120 + assert backups["provider"] == "filesystem" + assert "sync_type" not in backups + + +@pytest.mark.parametrize("provider", ["filesystem", "cache"]) +def test_agfs_config_accepts_supported_v2_backup_providers(provider): + """AGFS config must accept each specified metadata provider.""" + config = AGFSConfig( + path="/tmp/ov-test", + backend="local", + backups={"provider": provider, "items": []}, + ) + + assert config.backups["provider"] == provider + + +@pytest.mark.parametrize( + ("backups", "expected"), + [ + ({"initial_partitions": 0, "items": []}, "initial_partitions"), + ({"initial_partitions": 1025, "items": []}, "initial_partitions"), + ({"checkpoint_interval_secs": 59, "items": []}, "checkpoint_interval_secs"), + ({"provider": "memory", "items": []}, "provider"), + ({"provider": "redis", "items": []}, "provider"), + ({"items": [{"name": " ", "backend": "memory"}]}, "name"), + ({"items": [{"name": "primary", "backend": "memory"}]}, "primary"), + ( + { + "items": [ + {"name": "backup-a", "backend": "memory"}, + {"name": "backup-a", "backend": "memory"}, + ] + }, + "unique", + ), + ], +) +def test_agfs_config_rejects_invalid_v2_backups(backups, expected): + """AGFS config must reject invalid V2 backup fields before Rust startup.""" + with pytest.raises(ValueError, match=expected): + AGFSConfig(path="/tmp/ov-test", backend="local", backups=backups) + + +def test_generate_plugin_config_strips_legacy_multiwrite_fields(tmp_path): + """Plugin config generation should strip legacy fields before calling Rust.""" explicit_backup_dir = tmp_path / "backup-local-no-mkdir" config = AGFSConfig( path=str(tmp_path), backend="local", backups={ - "retry_interval_ms": 1234, - "retry_backoff_base_ms": 55, + "sync_type": "async", + "write_ack_count": 1, + "write_ack_timeout_ms": 1000, + "write_concurrency": 2, "items": [ { "name": "local-explicit", "backend": "local", - "local": {"workspace": str(explicit_backup_dir)}, + "params": {"workspace": str(explicit_backup_dir)}, + "timeout": 10, + "operations": [{"operation": "write", "priority": 1}], + "excludes": [{"type": "FileExtensionPolicy", "extensions": [".tmp"]}], }, { "name": "local-default", @@ -558,12 +626,17 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path): plugins = _generate_plugin_config(config, tmp_path) mount_backups = plugins["localfs"]["config"]["backups"] - assert mount_backups["retry_interval_ms"] == 1234 - assert mount_backups["retry_backoff_base_ms"] == 55 + assert { + "sync_type", + "write_ack_count", + "write_ack_timeout_ms", + "write_concurrency", + }.isdisjoint(mount_backups) explicit_local, default_local, s3_backup = mount_backups["items"] assert explicit_local["backend"] == "localfs" assert explicit_local["params"]["local_dir"] == str(explicit_backup_dir) + assert {"timeout", "operations", "excludes"}.isdisjoint(explicit_local) assert not explicit_backup_dir.exists() assert default_local["backend"] == "localfs" @@ -705,6 +778,23 @@ def test_ragfs_binding_enables_runtime_for_queuefs_cache_backend(tmp_path): assert binding["cache"]["runtime_enabled"] is True +def test_ragfs_binding_enables_runtime_for_multiwrite_cache_provider(tmp_path): + """Multi-write cache metadata should enable Runtime without CacheFS.""" + agfs_config = AGFSConfig( + path=str(tmp_path), + backend="local", + backups={"provider": "cache", "items": []}, + ) + + binding = RagfsBindingConfig( + agfs=agfs_config, + cache=CacheConfig(provider="redis", params={}), + ).to_binding_dict() + + assert binding["cache"]["enabled"] is False + assert binding["cache"]["runtime_enabled"] is True + + def test_agfs_pathlock_config_validates_provider_and_expiry(tmp_path): """PathLock config validates Redis namespace and expiry boundaries.""" config = AGFSConfig( diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.py index 739f97899b..365a7f6f12 100644 --- a/tests/server/test_admin_api.py +++ b/tests/server/test_admin_api.py @@ -34,7 +34,7 @@ user_config_backup_uri, ) from openviking.service.core import OpenVikingService -from openviking.service.deletion import setup_deletion +from openviking.service.deletion import DeletionService, setup_deletion from openviking.service.task_store import ( SYSTEM_TASK_ACCOUNT_ID, SYSTEM_TASK_USER_ID, @@ -311,6 +311,28 @@ async def _wait_for_task(client: httpx.AsyncClient, task_id: str) -> dict: # ---- Account CRUD ---- +async def test_delete_account_files_disables_automatic_pathlock(monkeypatch): + """Account cleanup must bypass the deleted tree's automatic PathLock.""" + path = "/local/account-to-delete" + agfs = Mock() + agfs.rm = AsyncMock() + agfs.stat = AsyncMock(side_effect=AGFSNotFoundError(path)) + service = Mock() + service.viking_fs._async_agfs = agfs + tracker = Mock() + monkeypatch.setattr("openviking.service.deletion.get_task_tracker", lambda: tracker) + deletion = DeletionService( + service=service, + manager=Mock(), + service_loop=asyncio.get_running_loop(), + ) + + await deletion._delete_account_files("account-to-delete") + + agfs.rm.assert_awaited_once_with(path, recursive=True, auto_pathlock=False) + tracker.forget_account_tasks.assert_called_once_with("account-to-delete") + + @pytest_asyncio.fixture async def template_account(lightweight_admin_client): account_id = _uid() @@ -324,17 +346,31 @@ async def template_account(lightweight_admin_client): async def test_studio_file_configuration_permissions_revision_and_overrides( - lightweight_admin_client, lightweight_admin_app, template_account, tmp_path, monkeypatch, + lightweight_admin_client, + lightweight_admin_app, + template_account, + tmp_path, + monkeypatch, ): from openviking.config.scope import ConfigScope from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton account_id, admin_headers = template_account config_path = tmp_path / "ov.conf" - config_path.write_text(json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}})) + config_path.write_text( + json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}}) + ) monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", config_path) manager = lightweight_admin_app.state.fake_service.runtime_config_manager - await manager.patch_account(account_id, {"vlm": {"model": "override-model", "credentials": [{"provider": "openai", "api_key": "account-secret"}]}}) + await manager.patch_account( + account_id, + { + "vlm": { + "model": "override-model", + "credentials": [{"provider": "openai", "api_key": "account-secret"}], + } + }, + ) url = "/api/v1/admin/configuration" params = {"source": "file", "account_id": account_id} denied = await lightweight_admin_client.get(url, params=params, headers=admin_headers) @@ -355,15 +391,28 @@ async def test_studio_file_configuration_permissions_revision_and_overrides( assert override_response.json()["result"]["overrides"]["cluster"] == ["query_planner"] content = json.loads(result["content"]) content["rerank"] = { - "provider": "jev", "api_key": "jev-secret", "mode": "choice", "threshold": 0 + "provider": "jev", + "api_key": "jev-secret", + "mode": "choice", + "threshold": 0, } body = {"revision": result["revision"], "content": json.dumps(content)} - assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=admin_headers, json=body)).status_code == 403 - saved = await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body) + assert ( + await lightweight_admin_client.patch( + url, params={"source": "file"}, headers=admin_headers, json=body + ) + ).status_code == 403 + saved = await lightweight_admin_client.patch( + url, params={"source": "file"}, headers=root_headers(), json=body + ) assert saved.status_code == 200, saved.text assert saved.json()["result"]["restart_required"] assert json.loads(config_path.read_text())["rerank"]["provider"] == "jev" - assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body)).status_code == 400 + assert ( + await lightweight_admin_client.patch( + url, params={"source": "file"}, headers=root_headers(), json=body + ) + ).status_code == 400 settings = await manager.get_settings(ConfigScope.account(account_id)) assert settings["vlm"]["model"] == "override-model" @@ -987,12 +1036,10 @@ async def test_account_memory_templates_range_expressions_reach_file_body( fs = lightweight_admin_app.state.fake_service.viking_fs snapshot = await resolve_account_memory_registry(fs, account_id, MemoryTypeRegistry()) schema = snapshot.get("events") - context = ExtractContext( - [ - Message(id="m1", role="user", parts=[TextPart("Selected source message")]), - Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]), - ] - ) + context = ExtractContext([ + Message(id="m1", role="user", parts=[TextPart("Selected source message")]), + Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]), + ]) rendered = MemoryFileUtils.write( MemoryFile(memory_type="events", extra_fields={"ranges": "0", "summary": "summary"}), content_template=schema.content_template, @@ -1315,33 +1362,29 @@ async def test_account_memory_templates_locked_fields_cannot_be_modified( for key, value in defaults.items() if key not in editable ] - invalid.extend( - [ - {"unknown": "ignored?"}, - {"Description": "wrong key"}, - {"_updated_at": "forged"}, - {"_account_content_template": False}, - {"deployment_content_template": "forged"}, - {"enabled": 1}, - {"fields": [{"name": "renamed_field", "description": "changed"}]}, - {"description": None}, - {"description": " \n"}, - {"fields": [None]}, - ] - ) + invalid.extend([ + {"unknown": "ignored?"}, + {"Description": "wrong key"}, + {"_updated_at": "forged"}, + {"_account_content_template": False}, + {"deployment_content_template": "forged"}, + {"enabled": 1}, + {"fields": [{"name": "renamed_field", "description": "changed"}]}, + {"description": None}, + {"description": " \n"}, + {"fields": [None]}, + ]) for field in defaults["fields"]: invalid.extend( {"fields": [{"name": field["name"], key: None if value is not None else "changed"}]} for key, value in field.items() if key not in {"name", "description"} ) - invalid.extend( - [ - {"fields": [{"name": field["name"], "unknown": "ignored?"}]}, - {"fields": [{"name": field["name"], "description": ""}]}, - {"fields": [{"name": field["name"]}, {"name": field["name"]}]}, - ] - ) + invalid.extend([ + {"fields": [{"name": field["name"], "unknown": "ignored?"}]}, + {"fields": [{"name": field["name"], "description": ""}]}, + {"fields": [{"name": field["name"]}, {"name": field["name"]}]}, + ]) if memory_type == "events": invalid.extend( {"fields": [{"name": name, "description": "changed"}]} for name in ("goal", "ranges") @@ -2131,6 +2174,33 @@ async def test_list_users_for_default_account_without_users_file_returns_empty_l assert resp.json()["result"] == [] +async def test_list_accounts_with_watcher_uses_memory( + lightweight_admin_client: httpx.AsyncClient, + lightweight_admin_app: FastAPI, + monkeypatch: pytest.MonkeyPatch, +): + """A running watcher keeps list requests on the in-memory fast path.""" + loop = asyncio.get_running_loop() + watch_task = loop.create_future() + + async def _unexpected_read(_path: str): + raise AssertionError("list accounts should not read AGFS while watcher is running") + + manager = lightweight_admin_app.state.api_key_manager + manager._store._watch_task = watch_task + monkeypatch.setattr(manager._store, "_read_json", _unexpected_read) + try: + resp = await lightweight_admin_client.get("/api/v1/admin/accounts", headers=root_headers()) + users_resp = await lightweight_admin_client.get( + "/api/v1/admin/accounts/default/users", headers=root_headers() + ) + finally: + watch_task.cancel() + + assert resp.status_code == 200, resp.text + assert users_resp.status_code == 200, users_resp.text + + async def test_identity_settings_refreshes_a_stale_registry_on_demand( admin_client: httpx.AsyncClient, admin_app: FastAPI, @@ -2308,22 +2378,18 @@ async def fail_vectors(account_id, *, ctx): account_id=SYSTEM_TASK_ACCOUNT_ID, user_id=SYSTEM_TASK_USER_ID, ) - await queue.enqueue( - { - "task_id": user_cleanup.task_id, - "account_id": acct, - "user_id": "alice", - "target": {"account_id": acct, "user_id": "bob"}, - } - ) - await queue.enqueue( - { - "task_id": late_task_id, - "account_id": SYSTEM_TASK_ACCOUNT_ID, - "user_id": SYSTEM_TASK_USER_ID, - "target": {"account_id": acct, "user_id": "bob"}, - } - ) + await queue.enqueue({ + "task_id": user_cleanup.task_id, + "account_id": acct, + "user_id": "alice", + "target": {"account_id": acct, "user_id": "bob"}, + }) + await queue.enqueue({ + "task_id": late_task_id, + "account_id": SYSTEM_TASK_ACCOUNT_ID, + "user_id": SYSTEM_TASK_USER_ID, + "target": {"account_id": acct, "user_id": "bob"}, + }) skipped = await _wait_for_task(admin_client, late_task_id) assert skipped["status"] == "completed" assert skipped["result"] == {"deleted": not recreated, "stale": True} @@ -3129,10 +3195,7 @@ async def test_trusted_mode_admin_can_register_user_in_own_account( headers=trusted_headers(account=acct, user="alice", include_api_key=True), ) assert users.status_code == 200 - assert all( - user["api_key_available"] is False - for user in users.json()["result"] - ) + assert all(user["api_key_available"] is False for user in users.json()["result"]) rotated = await trusted_admin_client.post( f"/api/v1/admin/accounts/{acct}/users/bob/key", @@ -3449,12 +3512,10 @@ async def test_server_restart_requires_root_and_valid_revision( server_overrides = {"host": cli_host} lightweight_admin_app.state.server_config_overrides = server_overrides path.write_text( - json.dumps( - { - "server": server, - "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"}, - } - ) + json.dumps({ + "server": server, + "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"}, + }) ) monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", path) monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file_content", path.read_bytes()) diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index 06c941a65a..dfe92bb2b7 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -16,6 +16,7 @@ from openviking.server.api_keys import APIKeyManager from openviking.server.app import create_app from openviking.server.auth import get_request_context, resolve_identity +from openviking.server.auth.plugins.api_key import _api_key_root_can_access_path from openviking.server.auth.registry import get_registry from openviking.server.config import ServerConfig, _is_localhost, validate_server_config from openviking.server.dependencies import set_service @@ -81,6 +82,20 @@ def _set_fake_task_tracker(): ROOT_KEY = "root-secret-key-for-testing-only-1234567890abcdef" +@pytest.mark.parametrize( + "path", + [ + "/api/v1/system/backend/sync-status", + "/api/v1/system/backend/sync-retry", + "/api/v1/system/sync/viking://resources", + "/api/v1/system/sync/viking://resources/retry", + ], +) +def test_root_key_allowlist_rejects_removed_sync_paths(path: str): + """Removed sync paths must stay outside the API-key root allowlist.""" + assert not _api_key_root_can_access_path(path) + + def _make_request( path: str, headers: dict[str, str] | None = None, @@ -390,46 +405,6 @@ async def test_auth_on_multiple_endpoints(auth_client: httpx.AsyncClient): assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED" -async def test_admin_sync_route_accepts_root_key(auth_client: httpx.AsyncClient, auth_service): - """ROOT keys should be allowed to call the system sync admin route.""" - calls: list[str] = [] - - async def _fake_system_sync_status(uri: str, ctx): - calls.append(uri) - return {"path": uri, "entry_count": 1} - - auth_service.fs.system_sync_status = _fake_system_sync_status - - resp = await auth_client.get( - "/api/v1/system/sync/viking://resources", - headers={"X-API-Key": ROOT_KEY}, - ) - assert resp.status_code == 200 - assert resp.json()["result"] == {"path": "viking://resources", "entry_count": 1} - assert calls == ["viking://resources"] - - -async def test_admin_sync_route_rejects_user_key(auth_client: httpx.AsyncClient, user_key: str): - """Regular user keys must not access the system sync admin route.""" - resp = await auth_client.get( - "/api/v1/system/sync/viking://resources", - headers={"X-API-Key": user_key}, - ) - assert resp.status_code == 403 - assert resp.json()["error"]["code"] == "PERMISSION_DENIED" - - tenant_resp = await auth_client.get( - "/api/v1/fs/ls?uri=viking://", - headers={ - "X-API-Key": ROOT_KEY, - "X-OpenViking-Account": "default", - "X-OpenViking-User": "default", - }, - ) - assert tenant_resp.status_code == 403 - assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED" - - async def test_task_endpoints_require_auth(): """Task endpoints must reject unauthenticated callers before lookup/filtering.""" set_task_tracker(None) @@ -1113,6 +1088,45 @@ async def test_invalid_trusted_identity_does_not_block_later_registration( await plugin.shutdown() +async def test_trusted_identity_registration_skips_identity_loaded_from_registry(auth_service): + """A restarted plugin must not queue an identity already loaded from storage.""" + from openviking.server.auth.plugins import TrustedAuthPlugin + + config = ServerConfig(auth_mode="trusted", root_api_key=ROOT_KEY) + app = FastAPI() + app.state.config = config + first_plugin = TrustedAuthPlugin() + await first_plugin.initialize(app, auth_service, config) + try: + await first_plugin._api_key_manager.ensure_trusted_identities({"acme": {"alice"}}) + finally: + await first_plugin.shutdown() + + restarted_app = FastAPI() + restarted_app.state.config = config + restarted_plugin = TrustedAuthPlugin() + await restarted_plugin.initialize(restarted_app, auth_service, config) + try: + request = _make_request( + "/api/v1/resources", + headers={ + "X-OpenViking-Account": "acme", + "X-OpenViking-User": "alice", + }, + auth_enabled=False, + auth_mode="trusted", + ) + request.app.state.config = config + + await restarted_plugin.resolve_identity( + request, x_openviking_account="acme", x_openviking_user="alice" + ) + + assert restarted_plugin._pending == {} + finally: + await restarted_plugin.shutdown() + + @pytest.mark.parametrize( "interval,path", [(0, "/api/v1/resources"), (0, "/api/v1/admin/accounts"), (300, "/api/v1/admin/accounts")], @@ -1210,7 +1224,14 @@ async def _fail_once(manager, identities): await plugin.shutdown() -def test_trusted_identity_registration_config_rejects_invalid_limits(): +def test_trusted_identity_registration_config_allows_disabling(): + """A zero interval disables registration; negative values remain invalid.""" + assert ( + ServerConfig( + trusted_identity_flush_interval_seconds=0 + ).trusted_identity_flush_interval_seconds + == 0 + ) with pytest.raises(ValueError): ServerConfig(trusted_identity_flush_interval_seconds=-1) with pytest.raises(ValueError): diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.py index d0e78b960d..d02d4a7c24 100644 --- a/tests/server/test_server_health.py +++ b/tests/server/test_server_health.py @@ -150,82 +150,24 @@ async def test_system_status(client: httpx.AsyncClient): assert body["result"]["initialized"] is True -async def test_backend_sync_status_endpoint(client: httpx.AsyncClient, service): - calls: list[str] = [] - - async def _fake_system_sync_status(uri: str, ctx): - calls.append(uri) - assert ctx is not None - return {"path": uri, "entry_count": 1} - - service.fs.system_sync_status = _fake_system_sync_status - - resp = await client.post( - "/api/v1/system/backend/sync-status", - json={"uri": "viking://resources"}, - ) - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "ok" - assert body["result"] == {"path": "viking://resources", "entry_count": 1} - assert calls == ["viking://resources"] - - -async def test_backend_sync_retry_endpoint(client: httpx.AsyncClient, service): - calls: list[str] = [] - - async def _fake_system_sync_retry(uri: str, ctx): - calls.append(uri) - assert ctx is not None - return {"path": uri, "retried": 2, "failed": 0} - - service.fs.system_sync_retry = _fake_system_sync_retry - - resp = await client.post( - "/api/v1/system/backend/sync-retry", - json={"uri": "viking://resources"}, - ) - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "ok" - assert body["result"] == {"path": "viking://resources", "retried": 2, "failed": 0} - assert calls == ["viking://resources"] - - -async def test_admin_sync_status_route(client: httpx.AsyncClient, service): - calls: list[str] = [] - - async def _fake_system_sync_status(uri: str, ctx): - calls.append(uri) - assert ctx is not None - return {"path": uri, "entry_count": 3} - - service.fs.system_sync_status = _fake_system_sync_status - - resp = await client.get("/api/v1/system/sync/viking://resources") - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "ok" - assert body["result"] == {"path": "viking://resources", "entry_count": 3} - assert calls == ["viking://resources"] - - -async def test_admin_sync_retry_route(client: httpx.AsyncClient, service): - calls: list[str] = [] - - async def _fake_system_sync_retry(uri: str, ctx): - calls.append(uri) - assert ctx is not None - return {"path": uri, "retried": 4, "failed": 1} - - service.fs.system_sync_retry = _fake_system_sync_retry - - resp = await client.post("/api/v1/system/sync/viking://resources/retry") - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "ok" - assert body["result"] == {"path": "viking://resources", "retried": 4, "failed": 1} - assert calls == ["viking://resources"] +@pytest.mark.parametrize( + ("method", "path", "json"), + [ + ("POST", "/api/v1/system/backend/sync-status", {"uri": "viking://resources"}), + ("POST", "/api/v1/system/backend/sync-retry", {"uri": "viking://resources"}), + ("GET", "/api/v1/system/sync/viking://resources", None), + ("POST", "/api/v1/system/sync/viking://resources/retry", None), + ], +) +async def test_removed_sync_routes_return_404( + client: httpx.AsyncClient, + method: str, + path: str, + json: dict | None, +): + """Removed V1 sync routes must stay unregistered.""" + resp = await client.request(method, path, json=json) + assert resp.status_code == 404 async def test_process_time_header(client: httpx.AsyncClient): @@ -253,15 +195,16 @@ async def test_404_for_unknown_route(client: httpx.AsyncClient): async def test_lifespan_shutdown_ignores_cancelled_service_close(monkeypatch): class _Service: + def __init__(self): + self.viking_fs = object() + self._queue_manager = None + async def initialize(self): pass async def apply_agent_evolution_config(self): pass - viking_fs = None - _queue_manager = None - async def close(self): raise asyncio.CancelledError("shutdown") @@ -323,9 +266,6 @@ class MockVikingFS: async def ls(self, path, ctx=None): return [] - async def system_sync_status(self, uri, ctx=None): - return {"path": uri, "entry_count": 0} - def _get_vector_store(self): class MockVectorStore: async def health_check(self): @@ -344,6 +284,10 @@ class MockService: "openviking_cli.utils.ollama.detect_ollama_in_config", lambda config: (False, None, None), ) + monkeypatch.setattr( + "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance", + lambda: SimpleNamespace(embedding=SimpleNamespace(get_embedder=lambda: None)), + ) app = create_app(config=ServerConfig(), service=service) transport = httpx.ASGITransport(app=app) @@ -354,7 +298,7 @@ class MockService: assert body["status"] == "ready" assert body["checks"]["agfs"]["status"] == "ok" assert body["checks"]["agfs"]["checks"]["filesystem"] == "ok" - assert body["checks"]["agfs"]["checks"]["multiwrite_sync"] == "ok" + assert body["checks"]["agfs"]["checks"] == {"filesystem": "ok"} async def test_slow_init_does_not_block_health(monkeypatch): @@ -386,6 +330,7 @@ class MockService: def __init__(self): self._initialized = False self.viking_fs = object() + self._queue_manager = None async def initialize(self): self._initialized = True diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 7e8484014c..08444234cc 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -256,6 +256,14 @@ def test_generic_code_hosting_domains_include_supported_platforms(): ] +def test_example_config_loads_as_runtime_config(): + example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example" + + config = OpenVikingConfig.from_dict(load_json_config(example_path)) + + assert config.storage.agfs.backend == "local" + + def test_example_code_hosting_domains_match_runtime_defaults(): example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example" example_text = example_path.read_text(encoding="utf-8")