From 9e2619bbb27b258b325215e2d3b73daea0fece3a Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:08:43 +0800 Subject: [PATCH 1/4] fix #446: fence FsyncCoordinator against truncation, clamp durable/persisted index, fix purge order - FsyncCoordinator generation-fences against truncation races - remove_range clamps durable_index/persisted_index post-truncation - fix purge ordering relative to durable_index advance - fix flaky snapshot_transfer_does_not_block_apply_embedded test: `since` baseline was captured after the 80-entry write loop, racing against the async snapshot+purge task that can complete mid-loop --- .dockerignore | 4 +- .../src/storage/buffered_raft_log.rs | 180 +++++++++++------ .../concurrent_fsync_test.rs | 4 + .../drain_fsync_test.rs | 161 +++++++++++---- .../persisted_index_clamp_test.rs | 188 ++++++++++++++++++ .../process_crash_safety_test.rs | 102 ++++++++++ .../replace_range_fsync_test.rs | 88 ++++++++ .../truncation_fsync_fence_test.rs | 101 ++++++++++ .../src/storage/fsync_coordinator.rs | 27 ++- .../src/storage/fsync_coordinator_test.rs | 52 +++++ .../test_utils/mock/mock_storage_engine.rs | 51 +++++ d-engine-core/src/watch/mod.rs | 1 + ..._transfer_does_not_block_apply_embedded.rs | 36 +++- .../performance_test.rs | 14 +- .../storage_buffered_raft_log/stress_test.rs | 9 +- examples/single-node-expansion/Makefile | 25 ++- examples/single-node-expansion/config/n1.toml | 52 ++++- .../three-nodes-standalone/docker/Dockerfile | 4 +- examples/three-nodes-standalone/src/main.rs | 9 +- 19 files changed, 978 insertions(+), 130 deletions(-) create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs diff --git a/.dockerignore b/.dockerignore index 2a8dfc95..f20b5623 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ **/target/ .git/ -examples/ +examples/* +!examples/three-nodes-standalone +!examples/client-usage-standalone diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 713286e9..b841238e 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -197,6 +197,15 @@ impl TermSegments { /// on tokio worker threads or the inbound event loop. #[derive(Debug)] pub enum IOTask { + /// Persist entries on the IO thread. `append_entries()` sends this and + /// awaits `done` — replaces the old inline `persist_entries()` call that + /// ran on the caller's own task (raft-core-loop), which could block + /// behind the IO thread's own concurrent fsync. + Persist { + entries: Vec, + done: oneshot::Sender>, + }, + /// Atomically truncate from `truncate_from` then persist `new_entries`. /// Conflict-resolution path: truncate + write are a single atomic IO unit. /// `done` is signalled after the IO thread finishes the replace so callers @@ -258,6 +267,10 @@ where // Raft must not tell a client or a peer a write is safe ahead of this point, // regardless of what's already visible in `entries`. pub(crate) durable_index: AtomicU64, + // Highest index handed to the storage engine (page cache), not yet + // fsynced. Set by append_entries()'s synchronous persist_entries() call. + // Lets the IO thread know what to fsync without re-scanning/re-writing. + persisted_index: AtomicU64, // The next index to be allocated pub(crate) next_id: AtomicU64, @@ -465,10 +478,20 @@ where } self.insert_to_memory(&entries); - // Signal IO thread to persist. Multiple concurrent notify_one() calls - // while the IO thread is busy coalesce into one wakeup — no per-write - // kernel cond_signal. IO thread reads from SkipMap via max_index. - self.write_notify.notify_one(); + + // Route the actual write through the IO thread — never call + // persist_entries() inline from this task. + // Still blocks the caller until truly persisted. + let (done_tx, done_rx) = oneshot::channel(); + self.command_sender + .send(IOTask::Persist { + entries, + done: done_tx, + }) + .map_err(|e| NetworkError::SingalSendFailed(format!("Persist send failed: {e:?}")))?; + done_rx + .await + .map_err(|_| NetworkError::SingalSendFailed("Persist done channel closed".into()))??; Ok(()) } @@ -657,9 +680,8 @@ where self.purge_prefix(cutoff_index); // Purged entries are backed by the snapshot; treat cutoff as durable. - // fetch_max is monotonic — avoids racing fsync_coordinator's concurrent - // advance on the raft-io thread — and this fires LogFlushed consistently - // with every other durable_index advancement in this file. + // Must run after purge_prefix() — advance_durable_and_notify() validates + // against last_purged_index, which purge_prefix() just established. self.advance_durable_and_notify(cutoff_index.index); // Route purge through the IO thread so it never blocks the inbound event loop. @@ -839,6 +861,7 @@ where last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), + persisted_index: AtomicU64::new(disk_len), next_id: AtomicU64::new(disk_len + 1), write_notify: Arc::new(Notify::new()), command_sender: command_sender.clone(), @@ -956,9 +979,7 @@ where if should_break { break; } } _ = safety_timer.tick() => { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = Self::persist_pending_range(&this, start, end, &mut pending_max, "safety-net").await; + Self::fold_persisted_watermark(&this, &mut pending_max); if pending_max > 0 { this.fsync_coordinator.submit(&this, pending_max, vec![]); @@ -969,38 +990,14 @@ where } } - /// Writes entries in `(from, to]` that haven't reached page cache yet - /// (no fsync). Advances `pending_max` on success; propagates the error - /// as-is on failure — whether to notify any waiting `Flush` caller is - /// left to the caller. - async fn persist_pending_range( + /// Folds `persisted_index` (set by `append_entries()`'s synchronous + /// write) into `pending_max`, so the IO thread still dispatches fsync + /// for it even though writing is no longer this thread's job. + fn fold_persisted_watermark( this: &Arc, - from: u64, - to: u64, pending_max: &mut u64, - ctx: &str, - ) -> Result<()> { - if this.is_poisoned() { - return Err(Error::Fatal("raft log storage is poisoned".to_string())); - } - - if from > to { - return Ok(()); - } - let entries = this.get_entries_range(from..=to)?; - if entries.is_empty() { - return Ok(()); - } - this.log_store - .persist_entries(entries) - .await - .inspect(|_| { - *pending_max = (*pending_max).max(to); - }) - .inspect_err(|e| { - error!("{ctx} persist_entries failed: {e:?}"); - this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); - }) + ) { + *pending_max = (*pending_max).max(this.persisted_index.load(Ordering::Acquire)); } async fn run_batch_turn( @@ -1010,15 +1007,7 @@ where mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); - let mut persist_failed = false; - if let Err(e) = Self::persist_pending_range(this, start, end, pending_max, "batch").await { - for reply in replies.drain(..) { - let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); - } - persist_failed = true; - } + Self::fold_persisted_watermark(this, pending_max); // `seen_shutdown` is not a gate here — regardless of whether the // caller already knows shutdown is happening, any commands still @@ -1043,13 +1032,6 @@ where } } - if !replies.is_empty() && !persist_failed { - let start = *pending_max + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = - Self::persist_pending_range(this, start, end, pending_max, "batch catch-up").await; - } - this.fsync_coordinator.submit(this, *pending_max, replies); *pending_max = 0; if seen_shutdown { @@ -1078,6 +1060,33 @@ where IOTask::Shutdown => { unreachable!("Shutdown is always filtered out before reaching handle_non_write_cmd") } + IOTask::Persist { entries, done } => { + if this.is_poisoned() { + let _ = done.send(Err(Error::Fatal("raft log storage is poisoned".into()))); + return true; // signal batch_processor to exit — disk state is untrusted + } + let max_idx = entries.last().map(|e| e.index).unwrap_or(0); + let result = this.log_store.persist_entries(entries).await; + if let Err(ref e) = result { + error!("IOTask::Persist failed (fatal): {e:?}"); + this.mark_poisoned_and_notify(format!("Persist failed: {e:?}")); + let _ = done.send(result); + return true; // signal batch_processor to exit — disk state is corrupted + } + if max_idx > 0 { + let current_bound = this + .max_index + .load(Ordering::Acquire) + .max(this.last_purged_index.load(Ordering::Acquire)); + let safe_max_idx = max_idx.min(current_bound); + if safe_max_idx > 0 { + this.persisted_index.fetch_max(safe_max_idx, Ordering::AcqRel); + this.fsync_coordinator.submit(this, safe_max_idx, vec![]); + } + } + let _ = done.send(result); + false // write succeeded, storage still trustworthy — keep the IO thread running + } IOTask::ReplaceRange { truncate_from, new_entries, @@ -1101,6 +1110,7 @@ where } if max_idx > 0 { *pending_max = (*pending_max).max(max_idx); + this.fsync_coordinator.submit(this, max_idx, vec![]); } let _ = done.send(result); false @@ -1148,6 +1158,7 @@ where self.entries.write().clear(); self.durable_index.store(0, Ordering::Release); + self.persisted_index.store(0, Ordering::Release); self.next_id.store(1, Ordering::Release); // Reset boundaries @@ -1222,17 +1233,31 @@ where } } - /// Advance `durable_index` to `new_durable` (monotonically) and send `LogFlushed`. + // The single choke point every reported max_index must pass through — + // re-validates against the current log boundary regardless of how many + // upstream call sites raced to produce this value. pub(super) fn advance_durable_and_notify( &self, - new_durable: u64, + reported_max: u64, ) { - let prev = self.durable_index.fetch_max(new_durable, Ordering::AcqRel); - if new_durable > prev + let current_max = self + .max_index + .load(Ordering::Acquire) + .max(self.last_purged_index.load(Ordering::Acquire)); + let safe_max = reported_max.min(current_max); + debug_assert!( + safe_max == reported_max, + "advance_durable_and_notify: reported_max {reported_max} exceeded current bound {current_max}, clamped" + ); + if safe_max == 0 { + return; + } + let prev = self.durable_index.fetch_max(safe_max, Ordering::AcqRel); + if safe_max > prev && let Some(ref tx) = self.log_flush_tx { let _ = tx.send(crate::InternalEvent::LogFlushed { - durable_index: new_durable, + durable_index: safe_max, }); } } @@ -1283,6 +1308,12 @@ where let (new_min, new_max) = self.remove_range_locked(&entries, range); self.min_index.store(new_min, Ordering::Release); self.max_index.store(new_max, Ordering::Release); + + self.persisted_index.fetch_min(new_max, Ordering::AcqRel); + self.durable_index.fetch_min(new_max, Ordering::AcqRel); + // Clamps pending_max and bumps generation, in that order — see + // fence_truncation()'s doc comment for why the order matters. + self.fsync_coordinator.fence_truncation(new_max); // `entries` guard drops here (end of scope) — write lock released. } @@ -1382,9 +1413,6 @@ where // then term (Acquire) always observe a consistent pair. self.last_purged_term.store(cutoff.term, Ordering::Release); self.last_purged_index.store(cutoff.index, Ordering::Release); - - // `entries` guard drops here — everything above is now visible together - // to any reader acquiring the read lock or loading these atomics after. } // Update the term index (completely lock-free) @@ -1439,6 +1467,14 @@ where pub fn is_empty(&self) -> bool { self.entries.read().is_empty() } + + #[cfg(test)] + pub(super) fn set_max_index_for_test( + &self, + value: u64, + ) { + self.max_index.store(value, Ordering::Release); + } } impl Drop for BufferedRaftLog @@ -1503,10 +1539,18 @@ mod id_allocation_test; #[path = "buffered_raft_log_test/performance_test.rs"] mod performance_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/persisted_index_clamp_test.rs"] +mod persisted_index_clamp_test; + #[cfg(test)] #[path = "buffered_raft_log_test/pipeline_overlap_test.rs"] mod pipeline_overlap_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/process_crash_safety_test.rs"] +mod process_crash_safety_test; + #[cfg(test)] #[path = "buffered_raft_log_test/quorum_durability_test.rs"] mod quorum_durability_test; @@ -1519,6 +1563,10 @@ mod raft_properties_test; #[path = "buffered_raft_log_test/remove_range_test.rs"] mod remove_range_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/replace_range_fsync_test.rs"] +mod replace_range_fsync_test; + #[cfg(test)] #[path = "buffered_raft_log_test/shutdown_test.rs"] mod shutdown_test; @@ -1531,6 +1579,10 @@ mod term_index_test; #[path = "buffered_raft_log_test/term_segments_test.rs"] mod term_segments_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/truncation_fsync_fence_test.rs"] +mod truncation_fsync_fence_test; + #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 178a9328..fce20bf3 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -313,6 +313,10 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + // advance_durable_and_notify() clamps against max_index — simulate a log + // that already has 150 entries, matching the highest value used below. + raft_log.set_max_index_for_test(150); + // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. raft_log.advance_durable_and_notify(150); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 838cc386..daf9a096 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -828,6 +828,9 @@ async fn test_poisoned_skips_purge() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); + // advance_durable_and_notify() clamps against max_index — simulate a log + // that already has the entry this test purges up to. + raft_log.set_max_index_for_test(1); raft_log.poisoned.store(true, Ordering::SeqCst); let result = raft_log.purge_logs_up_to(LogId { term: 1, index: 1 }).await; @@ -843,13 +846,21 @@ async fn test_poisoned_skips_purge() { /// 2026-07-19 — `run_batch_turn`'s drain loop now replies before returning, /// instead of silently dropping the oneshot sender). /// -/// Ordering is made deterministic (not timing-sensitive) by gating the IO -/// thread inside its first `persist_entries()` call. While it's blocked, an +/// Ordering is made deterministic (not timing-sensitive) by gating the base +/// entries' `persist_entries()` call — `append_entries()` now calls it +/// synchronously, so the base append is spawned as its own task and blocks +/// there instead of returning immediately. While it's blocked, an /// `IOTask::Flush` is sent directly (guaranteed FIFO-first) followed by a /// conflict-triggering `filter_out_conflicts_and_append` call (sends -/// `IOTask::ReplaceRange` second). Releasing the gate lets `run_batch_turn` -/// drain both in one pass, in that order. -#[tokio::test] +/// `IOTask::ReplaceRange` second). Releasing the gate lets the base append +/// finish and `run_batch_turn` drain both queued commands in one pass, in +/// that order. +/// +/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous +/// `std::sync::mpsc::Receiver::recv()`, which would otherwise freeze the +/// single default executor thread that the spawned base-append task, the +/// conflict task, and this test body all need to share. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() { let (gate_tx, gate_rx) = std::sync::mpsc::channel::<()>(); let gate_rx = std::sync::Mutex::new(Some(gate_rx)); @@ -900,30 +911,34 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); - // Base entries land in memory synchronously; the IO thread wakes and - // immediately blocks inside the gated persist_entries() call, before it - // ever drains the command queue. - raft_log - .append_entries(vec![ - Entry { - index: 1, - term: 1, - payload: None, - }, - Entry { - index: 2, - term: 1, - payload: None, - }, - Entry { - index: 3, - term: 1, - payload: None, - }, - ]) - .await - .unwrap(); - sleep(Duration::from_millis(20)).await; // let the IO thread reach the gate + // Base entries land in memory synchronously (before the gate), then + // append_entries() blocks inside its own gated persist_entries() call — + // spawned so the rest of this test can proceed while it's stuck there. + let base_append_task = tokio::spawn({ + let raft_log = raft_log.clone(); + async move { + raft_log + .append_entries(vec![ + Entry { + index: 1, + term: 1, + payload: None, + }, + Entry { + index: 2, + term: 1, + payload: None, + }, + Entry { + index: 3, + term: 1, + payload: None, + }, + ]) + .await + } + }); + sleep(Duration::from_millis(20)).await; // let it reach the gate // Send Flush directly — guarantees it's enqueued before the ReplaceRange // sent below, so it's the one already sitting in `replies` when the @@ -955,6 +970,12 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() sleep(Duration::from_millis(50)).await; // let the ReplaceRange send land gate_tx.send(()).unwrap(); + timeout(Duration::from_secs(2), base_append_task) + .await + .expect("base append task must not hang") + .expect("base append task must not panic") + .expect("base append must succeed once the gate releases"); + let flush_result = timeout(Duration::from_secs(2), flush_rx) .await .expect("flush reply must not hang"); @@ -1045,7 +1066,7 @@ async fn test_poisoned_survives_reset() { /// A `persist_entries()` (page-cache write) failure poisons the log, exactly /// like an fsync failure does — these are two independent failure surfaces -/// (see `persist_pending_range` vs `FsyncCoordinator::run_until_caught_up`) +/// (see `IOTask::Persist` vs `FsyncCoordinator::run_until_caught_up`) /// and both must reach the same fatal outcome. /// /// Without this test, a bug that only wires up ONE of the two poisoning @@ -1073,18 +1094,20 @@ async fn test_persist_entries_failure_poisons() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Triggers the IO thread's persist_pending_range call, which hits the - // mock's first (failing) persist_entries() — this is the - // persist_pending_range poisoning path, NOT FsyncCoordinator's. - raft_log + // append_entries() routes the write through IOTask::Persist and awaits + // the IO thread's reply — a persist failure now surfaces synchronously, + // right here, not discovered later by some other task. + let result = raft_log .append_entries(vec![Entry { index: 1, term: 1, payload: None, }]) - .await - .unwrap(); - sleep(Duration::from_millis(20)).await; // let the IO thread process it + .await; + assert!( + result.is_err(), + "a persist_entries() failure must surface synchronously from append_entries()" + ); assert!( raft_log.is_poisoned(), @@ -1106,6 +1129,70 @@ async fn test_persist_entries_failure_poisons() { ); } +/// `IOTask::Persist`'s own `is_poisoned()` guard (top of its handler, on the +/// IO thread) is a *different* check from `append_entries()`'s caller-side +/// fast-fail (line ~471) — that one only protects writes submitted *after* +/// poisoning already happened. This test targets the IO-thread-side guard +/// specifically, for a `Persist` task that was already queued *before* the +/// log got poisoned by something else (e.g. a concurrent ReplaceRange/Purge +/// failure): send `IOTask::Persist` directly through `command_sender`, +/// bypassing `append_entries()` entirely. Uses a plain always-succeeds mock +/// (`with_id`, no call-count requirement) — if the IO-thread-side guard is +/// missing or removed, `persist_entries()` would run and `done` would carry +/// `Ok(())` instead of the expected "...poisoned..." error, which the +/// `other => panic!` arm below catches either way. +#[tokio::test] +async fn test_poisoned_rejects_queued_persist_task() { + let storage = Arc::new(MockStorageEngine::with_id( + "poisoned_rejects_queued_persist_task".into(), + )); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + // Poisoned by something unrelated to this Persist task — simulated + // directly, same as the other `test_poisoned_skips_*` tests in this file. + raft_log.poisoned.store(true, Ordering::SeqCst); + + let (done_tx, done_rx) = tokio::sync::oneshot::channel(); + raft_log + .command_sender + .send(IOTask::Persist { + entries: vec![Entry { + index: 1, + term: 1, + payload: None, + }], + done: done_tx, + }) + .expect("IO thread must still be alive to receive the task"); + + let result = done_rx.await.expect("IO thread must reply, not drop the sender"); + match result { + Err(Error::Fatal(msg)) => assert!( + msg.contains("poisoned"), + "expected the poisoned short-circuit to fire before persist_entries() \ + was ever called, got: {msg}" + ), + other => panic!( + "expected Err(Fatal(\"...poisoned...\")), got: {other:?} — this means \ + the IO-thread-side is_poisoned() guard did not fire and \ + persist_entries() ran anyway", + ), + } +} + /// If `notify_fatal`'s underlying channel is already closed when a failure /// happens, the node must not fail *silently* — poisoned must still end up /// `true`, and the failure must be visible somewhere (log line), even though diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs new file mode 100644 index 00000000..e91d3cb5 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -0,0 +1,188 @@ +//! `persisted_index` must never claim a follower has written more to its +//! storage engine than what its log actually contains right now. +//! +//! Scenario: a follower has replicated entries 1-10 from an old leader and +//! synchronously written them to its storage engine (page cache), but hasn't +//! fsynced yet. A new leader is elected, finds entries 2-10 don't match its +//! own history, and tells the follower to truncate everything from index=2 +//! onward — the follower's real log now only has index=1. The new leader then +//! sends one brand-new entry that happens to land at index=2 again (different +//! content, new term). +//! +//! `persisted_index` only ever moves up (`fetch_max`), so without clamping it +//! on truncation, it would still remember "wrote up to 10" from before the +//! truncation — a stale high-water mark that the small index=2 write can't +//! pull back down. The next disk sync would then advertise `durable_index=10` +//! to the rest of the engine, even though the follower's log — and its +//! storage engine — genuinely only holds entries 1 and 2. A power loss at +//! that moment would prove the claim false: the follower reboots with only +//! [1, 2], not [1..=10], yet something upstream may already have acted on +//! "this follower is durable through 10" (e.g. deciding it's safe to purge +//! earlier log entries elsewhere in the cluster). + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{ + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, +}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// `durable_index()` must never exceed `last_entry_id()` — it must never +/// claim durability for an index that doesn't exist in the log anymore. +#[tokio::test] +async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { + let ctx = BufferedRaftLogTestContext::new( + PersistenceStrategy::MemFirst, + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // isolate from the safety-net timer + }, + "durable_index_never_exceeds_log_after_truncation_and_resync", + ); + + // Old leader (term=1) replicates entries 1..=10. append_entries() persists + // them to the storage engine synchronously, but nothing fsyncs them yet. + ctx.append_entries(1, 10, 1).await; + assert_eq!(ctx.raft_log.last_entry_id(), 10); + assert_eq!(ctx.raft_log.durable_index(), 0, "nothing fsynced yet"); + + // New leader (term=2) finds index=2 doesn't match its history (term=1 + // there, should be term=2) and truncates from index=2 onward, replacing + // it with one brand-new entry — real log becomes just [1, 2]. This goes + // through filter_out_conflicts_and_append's term-conflict slow path: + // remove_range(2..=MAX) (the clamp under test fires here, since it drops + // max_index from 10 down to 1) followed by inserting the new index=2. + ctx.raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]) + .await + .unwrap(); + assert_eq!( + ctx.raft_log.last_entry_id(), + 2, + "log truncated and replaced down to [1, 2]" + ); + + // Trigger a disk sync and give it time to complete. + ctx.raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // The follower must never advertise durability for an index it doesn't + // actually have. If persisted_index wasn't clamped down during the + // truncation, this would report 10 here — a lie. + assert!( + ctx.raft_log.durable_index() <= ctx.raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) — it must not \ + claim durability for entries the truncation already discarded", + ctx.raft_log.durable_index(), + ctx.raft_log.last_entry_id() + ); + assert_eq!( + ctx.raft_log.durable_index(), + 2, + "durable_index must reach the log's true end (2), not a stale pre-truncation watermark" + ); +} + +/// Different ordering from the test above: there, `remove_range`'s clamp ran +/// *before* anything else touched `persisted_index`. Here, a `Persist` task +/// dispatched *before* the truncation is still stuck on the IO thread (write +/// not yet reached the storage engine) when the truncation's own clamp runs — +/// and only *afterward* does that stale `Persist` complete and call +/// `persisted_index.fetch_max(10, ..)` (the line under review in +/// `handle_write_cmd`'s `IOTask::Persist` arm), using an index from entries +/// the truncation already discarded. `fetch_max` only ever moves up, so if +/// this call isn't fenced the same way `advance_durable_and_notify` fences a +/// stale fsync (see `truncation_fsync_fence_test.rs`), it silently +/// resurrects the clamp remove_range just applied. +/// +/// Scenario: +/// 1. Old leader (term=1) replicates entries 1..=10. `append_entries()` +/// inserts them into memory immediately, then blocks inside +/// `persist_entries()` on a gate — the write hasn't reached the storage +/// engine yet. +/// 2. New leader (term=2): index=2 conflicts. `filter_out_conflicts_and_append` +/// runs — `remove_range(2..=MAX)` (in-memory, synchronous, not routed +/// through the IO thread) executes and clamps immediately; the task then +/// blocks on the IO thread for its own queued `ReplaceRange`, which can't +/// run yet because the IO thread is still stuck on step 1's gate. +/// 3. Release the gate. The stale `Persist` for entries 1..=10 completes and +/// calls `persisted_index.fetch_max(10, ..)` — after the truncation's +/// clamp already ran, using entries that no longer exist. The queued +/// `ReplaceRange` runs next but does not re-clamp (its clamp already +/// fired once, in step 2, at truncation time). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() { + let (storage, persist_gate) = MockStorageEngine::not_durable_gated_persist( + "persisted_index_does_not_adopt_a_stale_persist_after_truncation".into(), + ); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + let append_task = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { raft_log.append_entries(entries).await }) + }; + // Let append_task reach the gate inside persist_entries(). + tokio::time::sleep(Duration::from_millis(50)).await; + + let truncate_task = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await + }) + }; + // Let truncate_task run remove_range()'s synchronous clamp and reach + // its own await point (queued behind the still-gated Persist). + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + raft_log.last_entry_id(), + 2, + "remove_range()'s in-memory truncation must be visible immediately, \ + without waiting for the gated Persist or the queued ReplaceRange" + ); + + // Release the stale Persist — it completes and calls + // persisted_index.fetch_max(10, ..) using now-discarded entries. + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + append_task.await.unwrap().unwrap(); + truncate_task.await.unwrap().unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + raft_log.persisted_index.load(Ordering::Acquire) <= raft_log.last_entry_id(), + "persisted_index ({}) must never exceed last_entry_id ({}) — the stale \ + Persist for entries 1..=10 must not be adopted after truncation shrank \ + the log to [1, 2]", + raft_log.persisted_index.load(Ordering::Acquire), + raft_log.last_entry_id() + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs new file mode 100644 index 00000000..3819e101 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -0,0 +1,102 @@ +//! Process-crash safety of entries counted toward quorum. +//! +//! `calculate_majority_matched_index` counts a leader's own write via +//! `last_entry_id()` — the in-memory SkipMap — as soon as `append_entries()` +//! returns. That's fine for power-loss safety (fsync is deliberately async, +//! see quorum_durability_test.rs) as long as the entry has at least reached the +//! storage engine (OS-managed page cache / WAL), which survives an ordinary +//! process crash even without fsync. +//! +//! These tests pin down whether `append_entries()` actually waits for the +//! storage engine (`LogStore::persist_entries`) before returning. Today it does +//! not — persistence happens later, asynchronously, on the IO thread — so an +//! entry can be quorum-eligible while a process crash between `append_entries()` +//! returning and the IO thread's next wakeup would lose it. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::{ + BufferedRaftLog, FlushPolicy, LogStore, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, RaftLog, StorageEngine, +}; + +/// `append_entries()` must not return before the entry reaches the storage +/// engine — otherwise a quorum-eligible write exists only in memory and is +/// lost on an ordinary process crash (not just power loss). +/// +/// Gates `LogStore::persist_entries()` so it never completes during the test. +/// Today, `append_entries()` only inserts into the in-memory SkipMap and +/// notifies the IO thread — it does not call `persist_entries()` itself — so +/// it returns immediately regardless of the gate, and the storage engine never +/// sees the entry. After the fix, `append_entries()` must call +/// `persist_entries()` synchronously before returning, so with the gate closed +/// it must still be pending. +/// +/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous +/// `std::sync::mpsc::Receiver::recv()` inside `persist_entries()`, which now +/// runs directly on whichever task calls `append_entries()`. On the default +/// single-threaded runtime that would freeze the only executor thread — +/// including this test's own `sleep()` below — for the gate's entire +/// lifetime, an unrelated deadlock, not the behavior under test. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_append_entries_waits_for_storage_engine_before_returning() { + let (storage, persist_gate) = + MockStorageEngine::not_durable_gated_persist("append_waits_for_storage_engine".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage.clone()), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + + let entry = Entry { + index: 1, + term: 1, + payload: None, + }; + + let append_task = tokio::spawn({ + let raft_log = raft_log.clone(); + async move { raft_log.append_entries(vec![entry]).await } + }); + + // Long enough that, if append_entries() were waiting on persist_entries(), + // it would still be pending; short enough to keep the suite fast. + tokio::time::sleep(Duration::from_millis(100)).await; + + // FIXED: append_entries() now routes the write through the IO thread + // (IOTask::Persist + oneshot) and does not return until it completes — + // with the gate closed, it must still be pending. + assert!( + !append_task.is_finished(), + "append_entries() must not return before persist_entries() completes" + ); + + // Ground truth: query the storage engine directly, not raft_log's own + // SkipMap-backed accessor (which would show the entry regardless). + assert!( + storage.log_store().entry(1).await.unwrap().is_none(), + "entry must not be visible in the storage engine while persist_entries() is gated" + ); + + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + append_task.await.unwrap().unwrap(); + + // append_entries() only returns after persist_entries() completes now, so + // the entry must already be visible — no polling needed. + assert!( + storage.log_store().entry(1).await.unwrap().is_some(), + "entry must be in the storage engine once append_entries() returns" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs new file mode 100644 index 00000000..4f0beb2d --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -0,0 +1,88 @@ +//! `IOTask::ReplaceRange` (term-conflict truncation, see +//! `filter_out_conflicts_and_append`'s slow path) writes to the storage engine +//! synchronously and bumps `pending_max`, but is dispatched through the +//! `receiver.recv()` => `cmd => { handle_non_write_cmd(...) }` arm of the IO +//! thread's select loop — a branch that, unlike `run_batch_turn`, never calls +//! `fsync_coordinator.submit()`. If no further `append_entries()` call arrives +//! afterward (which would separately trigger a `run_batch_turn` via +//! `write_notify`), the replaced entries sit "written but never fsync-submitted" +//! indefinitely — nothing but the idle-timer safety net would ever flush them. +//! +//! This test pins down that gap: it disables the safety net (a very long +//! `idle_flush_interval_ms`) so only the normal notify-driven path could +//! possibly advance `durable_index`, then proves it never does after a +//! term-conflict truncation with no subsequent append. + +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{FlushPolicy, PersistenceStrategy}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// A term-conflict truncation (`ReplaceRange`) must eventually become durable +/// even if no `append_entries()` call follows it. +/// +/// Today it does not: `ReplaceRange` is handled outside `run_batch_turn`, so +/// nothing submits fsync for it. Only the idle-timer safety net would catch +/// this — and this test disables that timer (60s interval, well beyond the +/// test's wait window) to isolate the notify-driven path from the safety net. +/// +/// RED (today): `durable_index()` never reaches `last_entry_id()` after the +/// truncation, because the replaced entries' fsync was never submitted. +#[tokio::test] +async fn test_replace_range_becomes_durable_without_a_following_append() { + let ctx = BufferedRaftLogTestContext::new( + PersistenceStrategy::MemFirst, + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe + }, + "replace_range_becomes_durable_without_a_following_append", + ); + + // Arrange: log [1,2,3] all term=1, explicitly flushed durable. + ctx.append_entries(1, 3, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 3, "baseline must be durable"); + + // Act: leader (term=2) sends entries that conflict at index=2 and extend + // the log to index=4. filter_out_conflicts_and_append's slow path detects + // the term mismatch at index=2, truncates [2,3], and replaces with + // [2,3,4] (term=2) via IOTask::ReplaceRange — with no append_entries() + // call afterward. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2), entry(3, 2), entry(4, 2)]) + .await + .unwrap(); + assert_eq!(result.unwrap().index, 4); + assert_eq!( + ctx.raft_log.last_entry_id(), + 4, + "memory must reflect the replace" + ); + + // Give the IO thread ample time to have submitted fsync, if anything + // besides the (disabled) safety net were going to do it. + tokio::time::sleep(Duration::from_millis(200)).await; + + // FIXED: ReplaceRange's handler now submits fsync directly instead of + // relying on a following append/notify or the safety net. + assert_eq!( + ctx.raft_log.durable_index(), + 4, + "ReplaceRange must submit fsync itself, without needing a following append" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs new file mode 100644 index 00000000..c2532224 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -0,0 +1,101 @@ +//! `FsyncCoordinator`'s `generation` fence protects against a physical fsync +//! whose result arrives after the world it was syncing no longer exists — but +//! today only `reset()` (full wipe) bumps `generation` via `fence_reset()`. +//! Term-conflict truncation (`filter_out_conflicts_and_append`'s slow path, +//! `remove_range` + `IOTask::ReplaceRange`) does not. +//! +//! Scenario: a follower has 10 entries synchronously written to its storage +//! engine but not yet fsynced — a physical fsync for "up to index 10" is +//! already dispatched and running in the background. Before that fsync +//! returns, a new leader tells the follower its log from index=2 onward is +//! wrong; the follower truncates and replaces it, ending up with only +//! entries [1, 2]. The in-flight fsync — which has no way to know any of +//! this happened — then completes and reports "index 10 is durable" anyway. +//! `durable_index` only moves up (`fetch_max`), so nothing afterward can +//! correct this: `durable_index()` gets stuck claiming durability for +//! entries [3..=10], which don't exist in this follower's log anymore. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::{ + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, +}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// `durable_index()` must never exceed `last_entry_id()` — a follower must +/// never claim durability for log entries a truncation has already discarded. +/// +/// RED (today): the stale in-flight fsync (dispatched for index=10, before +/// the truncation) is not fenced, and blindly advances `durable_index` to 10 +/// after the truncation has already shrunk the log to [1, 2]. +#[tokio::test] +async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { + // Gate closed: the first flush() call — for the original 10-entry batch — + // blocks here until we release it, letting us deterministically truncate + // the log while that fsync is still "in flight". + let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( + "durable_index_does_not_adopt_a_stale_fsync_after_truncation".into(), + ); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + + // Old leader (term=1) replicates entries 1..=10. append_entries() persists + // them synchronously; write_notify then wakes the IO thread, which + // dispatches a physical fsync for "up to index=10" — that fsync is now + // running in the background, blocked on flush_gate. + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + // Give the IO thread + blocking task time to reach the gated flush() call. + tokio::time::sleep(Duration::from_millis(50)).await; + + // New leader (term=2): index=2 conflicts, truncate and replace — the + // stale fsync (still blocked on the gate) has no way to observe this. + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await.unwrap(); + assert_eq!( + raft_log.last_entry_id(), + 2, + "log must be truncated and replaced down to [1, 2] before the stale fsync completes" + ); + + // Release the gate — the stale fsync (dispatched for index=10, before the + // truncation) now completes. + flush_gate.send(()).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + raft_log.durable_index() <= raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) — the stale \ + fsync for index=10 must not be adopted after truncation shrank the \ + log to [1, 2]", + raft_log.durable_index(), + raft_log.last_entry_id() + ); +} diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 47b74313..603a0c01 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -19,7 +19,11 @@ pub(super) struct FsyncCoordinator { inflight: AtomicBool, pending_max: AtomicU64, pending_replies: Mutex>>>, - generation: AtomicU64, // Bumped on every reset; fences out stale in-flight fsync results. + + // Fencing token (like Raft's `term`) for in-flight fsync results. Private — + // only bump via a fence_*() verb below, one per invalidating event. Never a + // value-passing variant (index math can under-fence, see fence_truncation()). + generation: AtomicU64, } impl FsyncCoordinator { @@ -156,13 +160,16 @@ impl FsyncCoordinator { } } + fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::AcqRel); + } + /// Called from reset_internal() before clearing in-memory state. /// Bumps generation to fence the in-flight physical flush (if any), /// AND drains anything already queued but not yet picked up by a /// flush round — that queued data was submitted before reset and /// must not be silently adopted by the next round. pub(super) fn fence_reset(&self) { - self.generation.fetch_add(1, Ordering::AcqRel); self.pending_max.store(0, Ordering::Release); let stale = std::mem::take(&mut *self.pending_replies.lock().unwrap()); for reply in stale { @@ -170,6 +177,22 @@ impl FsyncCoordinator { "stale fsync generation, superseded by reset".into(), ))); } + self.bump_generation(); + } + + /// Called from `remove_range()` before a truncation is applied. Bumps + /// `generation` to fence any fsync already in flight for data this + /// truncation is about to discard — mirrors `fence_reset()`, but does + /// NOT touch `pending_max`/`pending_replies`: unlike a full reset, + /// a truncation's own `IOTask::ReplaceRange` handler submits a fresh, + /// correct `max_index` for the surviving log right after this runs, + /// so there is nothing stale left to drain. + pub(super) fn fence_truncation( + &self, + new_max: u64, + ) { + self.pending_max.fetch_min(new_max, Ordering::AcqRel); + self.bump_generation(); } } diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 76cf0bf7..32f21ffd 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -341,6 +341,9 @@ fn test_run_until_caught_up_advances_durable_index_on_success() { ); let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — this test + // simulates a log that already has 5 entries, matching pending_max below. + raft_log.set_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); coord.pending_max.store(5, Ordering::Release); @@ -461,6 +464,53 @@ fn test_run_until_caught_up_discards_stale_generation_result_without_advancing() ); } +/// The other half of the fence: when `generation` at completion still +/// matches `generation` at the round's start (nothing fenced it while the +/// physical flush was running), the result must be accepted normally — +/// `durable_index` advances and queued replies resolve to `Ok`. +/// +/// Bumps `generation` twice before the round starts (so this isn't just +/// "stays at the default 0"), proving it's the *match*, not the specific +/// value, that matters. +/// +/// Expected: +/// - `durable_index()` advances to the round's `max_index`. +/// - The queued reply resolves to `Ok(())`. +#[test] +fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { + let (storage, _flush_call_count) = MockStorageEngine::not_durable( + "run_until_caught_up_accepts_result_when_generation_unchanged".into(), + ); + let coord = FsyncCoordinator::new(); + let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — matches pending_max below. + raft_log.set_max_index_for_test(5); + + // Two unrelated fences happened earlier — generation is 2, not 0 — before + // this round is even recorded as in flight. + coord.fence_reset(); + coord.fence_reset(); + assert_eq!(coord.generation.load(Ordering::Acquire), 2); + + let (tx, mut rx) = oneshot::channel::>(); + coord.inflight.store(true, Ordering::Release); + coord.pending_max.store(5, Ordering::Release); + coord.pending_replies.lock().unwrap().push(tx); + + // Nothing fences this round while it runs — generation stays at 2. + coord.run_until_caught_up(&raft_log); + + assert_eq!( + raft_log.durable_index.load(Ordering::Acquire), + 5, + "a matching generation must let the result advance durable_index normally" + ); + assert!( + rx.try_recv().expect("reply must have been answered").is_ok(), + "a matching generation must resolve queued replies as Ok, not Err" + ); +} + /// Multiple `submit()` calls made while a round is in flight are coalesced /// into a single subsequent physical `flush()` call by the same task — not /// one physical flush per `submit()` call. @@ -476,6 +526,8 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { ); let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — matches pending_max below. + raft_log.set_max_index_for_test(10); // Simulate two submit() calls that both lost the CAS while a round was // in flight — both just accumulated into the same pending state. diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index b21f91da..7990cf58 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -635,6 +635,57 @@ impl MockStorageEngine { (engine, tx) } + /// Create a MockStorageEngine where the first `persist_entries()` call blocks + /// until the returned sender fires. `flush()`/`is_write_durable()` are left at + /// their always-succeeds default (`configure_durable`) — this gate is only + /// about the write-to-storage-engine step, not fsync. + /// + /// Use this to make the ordering between `append_entries()` returning and the + /// entry actually reaching the storage engine deterministic (no sleep/race) — + /// see `process_crash_safety_test.rs`. + pub fn not_durable_gated_persist(id: String) -> (Self, std::sync::mpsc::Sender<()>) { + let (tx, rx) = std::sync::mpsc::channel::<()>(); + let rx = Mutex::new(Some(rx)); + + let mut mock_log_store = MockLogStore::new(); + let mut mock_meta_store = MockMetaStore::new(); + + Self::configure_mocks(&mut mock_log_store, &mut mock_meta_store, &id); + // persist_entries is gated below instead of via configure_persist_entries_success. + Self::configure_replace_range_success(&mut mock_log_store, &id); + Self::configure_purge_success(&mut mock_log_store); + Self::configure_reset_success(&mut mock_log_store, &id); + Self::configure_save_hard_state_success(&mut mock_meta_store, &id); + Self::configure_durable(&mut mock_log_store); + + let instance_id_ref = id.clone(); + mock_log_store.expect_persist_entries().returning(move |entries| { + // Only the first call blocks — take() leaves None for subsequent calls. + if let Some(gate) = rx.lock().unwrap().take() { + let _ = gate.recv(); // blocks until the test sends () + } + let mut data = MOCK_STORAGE_DATA.lock().unwrap(); + for entry in &entries { + let key = format!("{instance_id_ref}_entry_{}", entry.index); + let value = bincode::serialize(entry).unwrap(); + data.insert(key, value); + } + if let Some(last_entry) = entries.last() { + let key = format!("{instance_id_ref}_last_index"); + data.insert(key, last_entry.index.to_be_bytes().to_vec()); + } + Ok(()) + }); + + let engine = Self { + log_store: Arc::new(mock_log_store), + meta_store: Arc::new(mock_meta_store), + instance_id: id, + }; + + (engine, tx) + } + /// Configure `is_write_durable=true` and no-op flush (durable mock). fn configure_durable(log_store: &mut MockLogStore) { log_store.expect_is_write_durable().returning(|| true); diff --git a/d-engine-core/src/watch/mod.rs b/d-engine-core/src/watch/mod.rs index 0998f2c0..32fcc4ba 100644 --- a/d-engine-core/src/watch/mod.rs +++ b/d-engine-core/src/watch/mod.rs @@ -87,6 +87,7 @@ //! watcher_buffer_size: 256, //! enable_metrics: true, //! max_watcher_count: 5000, +//! heartbeat_interval_ms: 30_000, //! }; //! ``` //! diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs index b54a1080..666f1146 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs @@ -91,9 +91,20 @@ async fn test_snapshot_transfer_does_not_block_apply() -> Result<(), Box RETAINED_LOGS=8 (checked below), no node's log can cross a - // purge boundary before these writes land, so nothing earlier in the buffer can - // satisfy the match below — no extra gate is needed to make `since` safe. - // // This used to gate `since` on a `wait_for_snapshot` directory scan for the leader's // `.gz` file first, on the theory that "file exists" proves the snapshot is built. // It doesn't: `compress_directory` (default_state_machine_handler.rs) calls @@ -196,7 +202,6 @@ push_queue_size = 1 // made this test flake under CI load: RocksDB checkpoint export + tar/gzip // compression + metadata persist is genuinely sequential disk+CPU work that slows // down under contention. - let since = logs.lock().unwrap().len(); // The retained-log purge boundary — the actual signal this test needs, not just "a // snapshot file exists" (see comment above for why those differ). Emitted by @@ -208,14 +213,29 @@ push_queue_size = 1 // SNAPSHOT_THRESHOLD=64 > RETAINED_LOGS=8, the earliest possible snapshot on this // cluster already has last_included.index >= 64, so purge_upto_index is always > 0 // by the time this log line can appear at all. + // 60 x 500ms = 30s, not 15s: this test lives in the `multi-node-cluster-local` + // nextest group (throttled but not serialized, see .config/nextest.toml), and the + // log line polled below shares a process-global Mutex> with every other + // concurrently-running test in this binary (see log_capture.rs). Under full-suite + // load, RocksDB checkpoint export + tar/gzip (genuinely sequential CPU+disk work, + // see comment above) plus that shared-mutex contention can push real completion past + // 15s even though nothing is actually wrong — same root cause already documented in + // stress_test.rs's 30s bound. let mut purged = false; - for _ in 0..30 { + for _ in 0..60 { if logs_contain_globally_since(&logs, since, "purge_upto_index=") { purged = true; break; } tokio::time::sleep(Duration::from_millis(500)).await; } + if !purged { + eprintln!("=== DEBUG: captured logs since baseline writes ==="); + for line in logs.lock().unwrap()[since..].iter() { + eprintln!("{line}"); + } + eprintln!("=== END DEBUG ==="); + } assert!( purged, "Leader never logged a completed log purge — node 4 joining now would prove \ diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 0368f99a..453e17e7 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -225,16 +225,26 @@ async fn test_performance_benchmarks() { // Adjust test parameters according to the environment let operations = if is_ci { // CI environment uses a more relaxed threshold + // + // append_entries now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage + // engine before counting toward quorum) — this is an intentional + // correctness/speed tradeoff, not a regression. The old threshold + // (500) predates that fix. New floor leaves ~2x headroom below the + // observed ~318-328 ops/sec on a modern dev machine, keeping the + // 2:1 local:CI ratio from before. [ - ("append_entries", 500, 500.0), + ("append_entries", 500, 100.0), ("get_entries_range", 2500, 25000.0), ("entry_lookup", 5000, 100000.0), ("term_queries", 4000, 25000.0), ] } else { // Local environment uses a stricter threshold + // + // See CI-branch comment above — same #444 rationale. [ - ("append_entries", 1000, 1000.0), + ("append_entries", 1000, 200.0), ("get_entries_range", 5000, 50000.0), ("entry_lookup", 10000, 200000.0), ("term_queries", 8000, 50000.0), diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index f89ae55a..44972c8a 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -123,8 +123,15 @@ async fn test_high_concurrency_mixed_operations() { // Verify data integrity assert_eq!(ctx.raft_log.len(), 10000); + // append_entries() now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage engine + // before counting toward quorum) — an intentional correctness/speed + // tradeoff, not a regression. The old 10s bound predates that fix; + // observed wall-clock for this test's 10k concurrent writes is now + // 13-18s depending on machine load. New bound leaves real headroom + // above that range rather than chasing the exact number. assert!( - duration < Duration::from_secs(10), + duration < Duration::from_secs(30), "Operations took too long: {duration:?}" ); } diff --git a/examples/single-node-expansion/Makefile b/examples/single-node-expansion/Makefile index a31b2775..a00218c4 100644 --- a/examples/single-node-expansion/Makefile +++ b/examples/single-node-expansion/Makefile @@ -8,12 +8,35 @@ # =============================== LOG_LEVEL ?= debug + +# On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ +# compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 +# (Clang 16 lacks __builtin_ctzg/__builtin_clzg from LLVM 18+ SDK headers). +# brew --prefix resolves correctly on both Apple Silicon (/opt/homebrew) and +# Intel Mac (/usr/local). Silently no-ops when brew or a lib is absent. +SNAPPY_PREFIX := $(shell brew --prefix snappy 2>/dev/null) +LZ4_PREFIX := $(shell brew --prefix lz4 2>/dev/null) +ZSTD_PREFIX := $(shell brew --prefix zstd 2>/dev/null) +BREW_ROCKSDB_ENV := + +ifneq ($(SNAPPY_PREFIX),) +ifneq ($(wildcard $(SNAPPY_PREFIX)/lib),) + BREW_ROCKSDB_ENV += SNAPPY_LIB_DIR=$(SNAPPY_PREFIX)/lib +endif +endif +ifneq ($(LZ4_PREFIX),) + BREW_ROCKSDB_ENV += LZ4_LIB_DIR=$(LZ4_PREFIX)/lib +endif +ifneq ($(ZSTD_PREFIX),) + BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib +endif + # =============================== # Build Targets # =============================== build: @echo "Building release binary..." - cargo build --release --jobs 4 + $(BREW_ROCKSDB_ENV) cargo build --release --jobs 4 # =============================== # Single Node Bootstrap diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index bdb78715..712a3935 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -14,6 +14,8 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 +cmd_channel_capacity = 1024 +ordered_channel_capacity = 1024 [raft.election] election_timeout_min = 1000 @@ -23,27 +25,54 @@ election_timeout_max = 2000 default_policy = "LeaseRead" lease_duration_ms = 500 +[raft.read_actor] +channel_capacity = 10240 +max_drain = 2000 + +[raft.batching] +# Maximum number of commands to accumulate in a single batch during drain operations +max_batch_size = 200 + +[raft.metrics] +enable_backpressure = false +enable_batch = false + +[raft.backpressure] +max_pending_writes = 1000 +max_pending_reads = 500 + + [raft.persistence] +# strategy = "DiskFirst" strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 20 } } +flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } +# Maximum number of log entries to buffer in memory +# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] -enable = false -max_log_entries_before_snapshot = 10000 -retained_log_entries = 3 +enable = true +max_log_entries_before_snapshot = 5000 +retained_log_entries = 100 +cleanup_retain_count = 100 -# == Network Control Plane == +# == TTL Lease Configuration == +[raft.state_machine.lease] +cleanup_interval_ms = 1000 +max_cleanup_duration_ms = 1 + +# == Network Control Plane (voting, heartbeat, etc.) == [network.control] connection_window_size = 4_194_304 stream_window_size = 2_097_152 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 -http2_keep_alive_timeout_in_secs = 10 +http2_keep_alive_interval_in_secs = 15 # Slightly increase to reduce frequent keep-alives +http2_keep_alive_timeout_in_secs = 10 # Increase timeout +# New performance tuning parameters -# == Network Data Plane == +# == Network Data Plane (append_entries, etc.) == [network.data] connect_timeout_in_ms = 100 request_timeout_in_ms = 300 @@ -51,13 +80,16 @@ connection_window_size = 8_388_608 stream_window_size = 4_194_304 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 +http2_keep_alive_interval_in_secs = 15 # Same as control plane http2_keep_alive_timeout_in_secs = 10 - +# New data plane optimizations # == Server Transport (single listener serving every RPC type) == [network.server] concurrency_limit_per_connection = 100 # Increased for higher concurrent replication load max_concurrent_streams = 4096 # Increased to reduce stream creation overhead max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid Reset mitigation + +[storage] +unified_db = false diff --git a/examples/three-nodes-standalone/docker/Dockerfile b/examples/three-nodes-standalone/docker/Dockerfile index df52fc4b..e971495e 100644 --- a/examples/three-nodes-standalone/docker/Dockerfile +++ b/examples/three-nodes-standalone/docker/Dockerfile @@ -55,6 +55,8 @@ RUN apt-get update && \ iptables \ iproute2 \ libc6 \ + libfuse3-3 \ + fuse3 \ tzdata && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /var/run/sshd && \ @@ -85,4 +87,4 @@ COPY examples/three-nodes-standalone/docker/monitoring/promtail/config.yml /etc/ WORKDIR /app -CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] +CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT DB_PATH=/app/db/$ID RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index 4a0fd403..de1e1f83 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -17,7 +17,7 @@ use tracing_subscriber::Layer; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; -#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { let log_dir = env::var("LOG_DIR") .map_err(|_| "LOG_DIR environment variable not set") @@ -58,8 +58,11 @@ async fn main() { let (graceful_tx, graceful_rx) = watch::channel(()); // Start the server (wait for its initialization to complete) - let server_handler = - tokio::spawn(start_dengine_server(data_dir, config_path, graceful_rx.clone())); + let server_handler = tokio::spawn(start_dengine_server( + data_dir, + config_path, + graceful_rx.clone(), + )); // Wait for the server to initialize (adjust the waiting time according to the actual logic) tokio::time::sleep(Duration::from_secs(1)).await; From 340b518912151d4f5971b1961663ae4f0aefcb33 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:36:55 +0800 Subject: [PATCH 2/4] fix #446: gate commit quorum and follower ACKs on fsync-durable index (RPO=0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Leader's quorum contribution now uses durable_index, not last_entry_id (including single-voter clusters, which previously fell back to last_entry_id per fix #329 — RPO=0 is now mandatory there too). - Follower/learner AppendEntries ACKs are withheld until the node's own durable_index catches up (new PendingAck, released on LogFlushed). - Rewrote the gRPC AppendEntries forwarder (FuturesUnordered, no strict FIFO) to remove the head-of-line blocking that withheld ACKs would otherwise cause; added stuck-send detection (error log + metric). - Renamed → ; removed the dead single-variant / config and its example/bench TOML references. - Test coverage: quorum-durability unit tests, pending-ack dedup/boundary/ role-transition-drop-safety, forwarder ordering end-to-end, and a real- disk crash + quorum composition test. - Updated CHANGELOG and the throughput-optimization-guide for the new ack-latency-not-data-loss framing. --- CHANGELOG.md | 23 ++ benches/embedded-bench/config/n1.toml | 2 - benches/embedded-bench/config/n2.toml | 2 - benches/embedded-bench/config/n3.toml | 2 - benches/reports/v0.2.5/bench_report_v0.2.5.md | 3 +- d-engine-core/src/config/raft.rs | 53 +-- d-engine-core/src/lib.rs | 5 + d-engine-core/src/raft_role/follower_state.rs | 16 + .../src/raft_role/follower_state_test.rs | 335 +++++++++++++++++- d-engine-core/src/raft_role/leader_state.rs | 15 +- .../single_voter_commit_test.rs | 75 ++-- d-engine-core/src/raft_role/learner_state.rs | 13 + .../src/raft_role/learner_state_test.rs | 102 +++++- d-engine-core/src/raft_role/role_state.rs | 85 ++++- .../src/storage/buffered_raft_log.rs | 16 +- .../basic_operations_test.rs | 28 +- .../concurrent_fsync_test.rs | 72 ++-- .../concurrent_operations_test.rs | 5 +- .../drain_fsync_test.rs | 18 - .../durable_index_test.rs | 8 +- .../buffered_raft_log_test/edge_cases_test.rs | 6 +- .../flush_strategy_test.rs | 11 +- .../id_allocation_test.rs | 4 +- .../performance_test.rs | 51 +-- .../persisted_index_clamp_test.rs | 7 +- .../pipeline_overlap_test.rs | 4 +- .../process_crash_safety_test.rs | 3 +- .../quorum_durability_test.rs | 235 +++++++++--- .../raft_properties_test.rs | 10 +- .../remove_range_test.rs | 11 +- .../replace_range_fsync_test.rs | 3 +- .../buffered_raft_log_test/shutdown_test.rs | 8 +- .../buffered_raft_log_test/term_index_test.rs | 8 +- .../term_segments_test.rs | 3 +- .../truncation_fsync_fence_test.rs | 6 +- .../buffered_raft_log_test/worker_test.rs | 3 +- .../src/storage/fsync_coordinator_test.rs | 2 - d-engine-core/src/storage/raft_log.rs | 12 +- .../buffered_raft_log_test_helpers.rs | 11 +- .../src/network/grpc/grpc_raft_service.rs | 107 ++++-- .../network/grpc/grpc_raft_service_test.rs | 175 +++++++++ d-engine-server/src/node/builder_test.rs | 2 - .../src/test_utils/integration/mod.rs | 2 - d-engine-server/tests/common/mod.rs | 4 - .../crash_recovery_test.rs | 25 +- .../tests/storage_buffered_raft_log/mod.rs | 11 +- .../performance_test.rs | 9 +- .../quorum_crash_recovery_test.rs | 108 ++++++ .../storage_integration_test.rs | 3 +- .../storage_buffered_raft_log/stress_test.rs | 9 +- .../watch_membership_embedded.rs | 2 - .../docs/examples/three-nodes-standalone.md | 3 +- .../throughput-optimization-guide.md | 48 +-- .../server_guide/customize-storage-engine.md | 2 +- examples/single-node-expansion/config/n1.toml | 5 +- examples/single-node-expansion/config/n2.toml | 1 - examples/single-node-expansion/config/n3.toml | 1 - examples/sled-cluster/config/n1.toml | 2 - examples/sled-cluster/config/n2.toml | 2 - examples/sled-cluster/config/n3.toml | 2 - examples/three-nodes-embedded/README.md | 1 - .../three-nodes-standalone/config/n1.toml | 5 +- .../three-nodes-standalone/config/n2.toml | 5 +- .../three-nodes-standalone/config/n3.toml | 5 +- .../docker/config/n1.toml | 5 +- .../docker/config/n2.toml | 5 +- .../docker/config/n3.toml | 5 +- 67 files changed, 1269 insertions(+), 561 deletions(-) create mode 100644 d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d9012493..3299062d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ All notable changes to this project will be documented in this file. returns immediately, and entries arriving during an in-flight fsync are coalesced into the same physical disk flush. Storage-level group commit is restored without artificial batching windows. +- **🛑 Client-acknowledged writes could be lost on correlated power loss (#446)**: Raft commit quorum + counted the leader's own log contribution using its in-memory tail (`last_entry_id()`), not its + fsync-confirmed position (`durable_index()`) — a write could reach a majority-looking commit index, + and be acknowledged to the client, before enough replicas had actually synced it to disk. If those + nodes then lost power before their next fsync, the acknowledged write was gone. Fixed: leader quorum + calculation, follower `AppendEntries` ACK timing (a follower now withholds its response until its own + `durable_index` reaches the acknowledged entry), and single-voter clusters (previously exempted from + this class of fix, see #329) all gate on `durable_index`. RPO=0 for acknowledged writes is now a + mandatory invariant. Net effect: write acknowledgment latency now includes fsync time on a quorum of + replicas — see [Throughput Optimization Guide](./d-engine/src/docs/performance/throughput-optimization-guide.md) + for tuning `idle_flush_interval_ms`. + ### Changed - **MSRV raised to Rust 1.89**: The `data_dir` startup lock (prevents two node processes from @@ -65,6 +77,17 @@ All notable changes to this project will be documented in this file. - **`NodeBuilder` is no longer public** — use `EmbeddedEngine::start_custom`/`StandaloneEngine::run_custom` to plug in a custom storage engine or state machine. See [Migration Guide](./MIGRATION_GUIDE.md) for details. +- **⚠️ `[raft] ordered_channel_capacity` renamed to `max_pending_append_responses`** (#446): Follows the + gRPC `AppendEntries` forwarder rewrite (`FuturesUnordered`-based, no longer strict-FIFO) that shipped + alongside the durability fix above. Old field name is silently ignored, not an error — update existing + configs to the new name to keep the setting in effect. + +- **⚠️ `[raft.persistence] strategy` removed** (#446): `PersistenceStrategy` was a single-variant enum + (`MemFirst`) left over from #268; its only meaning now lives in whether an entry has reached + `durable_index`, which is no longer a configurable choice. Existing configs setting `strategy = + "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` + is the only persistence knob now. + --- ## [v0.2.4] - 2026-05-23 diff --git a/benches/embedded-bench/config/n1.toml b/benches/embedded-bench/config/n1.toml index 52055b93..8dec8c84 100644 --- a/benches/embedded-bench/config/n1.toml +++ b/benches/embedded-bench/config/n1.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/embedded-bench/config/n2.toml b/benches/embedded-bench/config/n2.toml index 56c9cf13..455effff 100644 --- a/benches/embedded-bench/config/n2.toml +++ b/benches/embedded-bench/config/n2.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/embedded-bench/config/n3.toml b/benches/embedded-bench/config/n3.toml index 701a1551..1296b03d 100644 --- a/benches/embedded-bench/config/n3.toml +++ b/benches/embedded-bench/config/n3.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5.md b/benches/reports/v0.2.5/bench_report_v0.2.5.md index 8e2bcb69..a6e08570 100644 --- a/benches/reports/v0.2.5/bench_report_v0.2.5.md +++ b/benches/reports/v0.2.5/bench_report_v0.2.5.md @@ -59,7 +59,7 @@ _(v0.2.5: 6-round average; v0.2.4: 4-round average; v0.2.3: 4-round average (Lea _(v0.2.5: 5-round average; v0.2.4: 5-round average; v0.2.3: 5-round average. All manually collected. 2026-07-12: 4-round average (conns=200, clients=200, Docker monitoring stack stopped).)_ | **Scenario** | **Metric** | **v0.2.3** | **v0.2.4** | **v0.2.5** | **Δ (v0.2.4→v0.2.5)** | **0712** | **Δ (v0.2.5→0712)** | -| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------ | ------------------- | +| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------- | ------------------- | | Single Client Write | Throughput | 6,421 ops/s | 5,245 ops/s | 5,234 ops/s | stable | 9,450 ops/s | **+80.5%** ✅ | | | Avg Latency | 0.155 ms | 0.190 ms | 0.190 ms | stable | 0.105 ms | **-44.6%** ✅ | | | p99 Latency | 0.200 ms | 0.235 ms | 0.237 ms | stable | 0.223 ms | -6.0% → | @@ -214,7 +214,6 @@ read_actor_channel_capacity = 10240 read_actor_max_drain = 2000 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.batching] diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 78e3a333..8e03f2f5 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -79,11 +79,15 @@ pub struct RaftConfig { #[serde(default = "default_cmd_channel_capacity")] pub cmd_channel_capacity: usize, - /// Ordered channel capacity for stream_append_entries ordering - /// Controls buffering of response receivers in FIFO order - /// Default value is set via default_ordered_channel_capacity() function - #[serde(default = "default_ordered_channel_capacity")] - pub ordered_channel_capacity: usize, + /// Max in-flight AppendEntries requests on `stream_append_entries` that can be + /// dispatched to the Raft loop and awaiting their response at once. Once this many + /// are pending, the stream stops reading new requests until one completes — this + /// bounds memory/task growth if this node's own durable_index stalls (RPO=0, #446). + /// Also used directly as the output channel's buffer size, since completed + /// responses can never outnumber in-flight requests. + /// Default value is set via default_max_pending_append_responses() function + #[serde(default = "default_max_pending_append_responses")] + pub max_pending_append_responses: usize, /// ReadActor configuration — tuning for the dedicated Eventual/LeaseRead fast path. #[serde(default)] @@ -141,7 +145,7 @@ impl Default for RaftConfig { auto_join: AutoJoinConfig::default(), snapshot_rpc_timeout_ms: default_snapshot_rpc_timeout_ms(), cmd_channel_capacity: default_cmd_channel_capacity(), - ordered_channel_capacity: default_ordered_channel_capacity(), + max_pending_append_responses: default_max_pending_append_responses(), read_actor: ReadActorConfig::default(), read_consistency: ReadConsistencyConfig::default(), backpressure: BackpressureConfig::default(), @@ -201,7 +205,7 @@ fn default_cmd_channel_capacity() -> usize { 1024 } -fn default_ordered_channel_capacity() -> usize { +fn default_max_pending_append_responses() -> usize { 1024 } @@ -817,27 +821,6 @@ impl Default for PromotionConfig { fn default_stale_learner_threshold() -> Duration { Duration::from_secs(300) } -/// Defines how Raft log entries are persisted and accessed. -/// -/// All strategies use a configurable [`FlushPolicy`] to control when memory contents -/// are flushed to disk, affecting write latency and durability guarantees. -/// -/// **Note:** Both strategies now fully load all log entries from disk into memory at startup. -/// The in-memory `SkipMap` serves as the primary data structure for reads in all modes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub enum PersistenceStrategy { - /// Memory-first persistence strategy. - /// - /// - **Write path**: On append, the log entry is first written to the in-memory `SkipMap` and - /// acknowledged immediately. Disk persistence happens asynchronously in the background, - /// governed by [`FlushPolicy`]. - /// - /// - **Read path**: Reads are always served from the in-memory `SkipMap`. - /// - /// - **Startup behavior**: All log entries are loaded from disk into memory at startup. - /// - MemFirst, -} /// Controls when in-memory logs should be flushed to disk. /// @@ -857,14 +840,6 @@ pub enum FlushPolicy { /// Configuration parameters for log persistence behavior #[derive(Serialize, Deserialize, Clone, Debug)] pub struct PersistenceConfig { - /// Strategy for persisting Raft logs - /// - /// This controls the trade-off between durability guarantees and performance - /// characteristics. The choice impacts both write throughput and recovery - /// behavior after node failures. - #[serde(default = "default_persistence_strategy")] - pub strategy: PersistenceStrategy, - /// Flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -886,11 +861,6 @@ pub struct PersistenceConfig { pub shutdown_timeout_ms: u64, } -/// Default persistence strategy (optimized for balanced workloads) -fn default_persistence_strategy() -> PersistenceStrategy { - PersistenceStrategy::MemFirst -} - /// Default flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -933,7 +903,6 @@ impl PersistenceConfig { impl Default for PersistenceConfig { fn default() -> Self { Self { - strategy: default_persistence_strategy(), flush_policy: default_flush_policy(), max_buffered_entries: default_max_buffered_entries(), shutdown_timeout_ms: default_shutdown_timeout_ms(), diff --git a/d-engine-core/src/lib.rs b/d-engine-core/src/lib.rs index 93333741..387324b2 100644 --- a/d-engine-core/src/lib.rs +++ b/d-engine-core/src/lib.rs @@ -173,6 +173,11 @@ pub(crate) fn if_higher_term_found( /// entries in the logs. If the logs have last entries with different terms, then the log with the /// later term is more up-to-date. If the logs end with the same term, then whichever log is longer /// is more up-to-date. +/// +/// #446: callers must pass the in-memory last-log-id (last_entry_id), never durable_index. +/// A node with an un-fsynced tail must still be able to reject a candidate whose log is +/// genuinely less up to date — voting eligibility and commit-durability are separate +/// concerns and must not share the same index source. pub(crate) fn is_target_log_more_recent( my_last_log_index: u64, my_last_log_term: u64, diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index c1e30ebf..f0dfde4e 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -7,6 +7,7 @@ use d_engine_proto::server::cluster::ClusterConfUpdateResponse; use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -43,6 +44,7 @@ use crate::RaftNodeConfig; use crate::Result; use crate::StateTransitionError; use crate::TypeConfig; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use crate::utils::cluster::error; use crate::utils::cluster_printer::print_role_transition_line; @@ -73,6 +75,10 @@ pub struct FollowerState { /// Last physically purged log index (inclusive) pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -463,6 +469,12 @@ impl RaftRoleState for FollowerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut( + &mut self + ) -> Option<&mut std::collections::BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl FollowerState { @@ -484,6 +496,7 @@ impl FollowerState { node_config.raft.election.election_timeout_max, )), node_config, + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new(false), _marker: PhantomData, last_purged_index: None, @@ -511,6 +524,7 @@ impl From<&CandidateState> for FollowerState { )), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, // scheduled_purge_upto: None, _marker: PhantomData, @@ -527,6 +541,7 @@ impl From<&LeaderState> for FollowerState { leader_state.node_config.raft.election.election_timeout_max, )), node_config: leader_state.node_config.clone(), + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new( leader_state.snapshot_in_progress.load(Ordering::SeqCst), ), @@ -548,6 +563,7 @@ impl From<&LearnerState> for FollowerState { )), node_config: learner_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: learner_state.last_purged_index, pending_purge_upto: learner_state.pending_purge_upto, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index d6ab39a8..2516c7fd 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -994,6 +994,16 @@ async fn test_handle_append_entries_success_from_new_leader() { "Should update commit_index" ); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + // Verify: Response with success=true let response = resp_rx.recv().await.expect("should receive response").unwrap(); assert!(response.is_success(), "Response should indicate success"); @@ -2885,12 +2895,12 @@ async fn test_follower_rejects_strong_consistency_reads() { // MemFirst ACK Tests // ============================================================================ -/// Follower ACKs leader immediately after memory write (MemFirst). +/// Follower withholds the AppendEntries ACK until its own durable_index catches up. /// -/// The IO thread continues to fsync asynchronously. Safety: before commit, -/// the leader's durable_index >= N (quorum uses durable_index). +/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the +/// claimed index is fsynced. LogFlushed releases the withheld response. #[tokio::test] -async fn test_follower_acks_immediately_after_memory_write() { +async fn test_follower_withholds_ack_until_durable() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); @@ -2937,9 +2947,306 @@ async fn test_follower_acks_immediately_after_memory_write() { .is_ok() ); - // MemFirst: ACK sent immediately, no waiting for fsync - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); + // RPO=0: the ACK is withheld while durable_index < claimed index (5). + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); +} + +/// #446: if `FollowerState` is dropped (role transition — e.g. a higher-term +/// AppendEntries or becoming a candidate) while it still holds a withheld ACK, the +/// pending sender must be dropped with it — the caller waiting on `resp_rx` must see +/// the channel close, not hang forever and not receive a stale success. +/// +/// This is the actual mechanism #446's design relies on for role-transition safety +/// (see the design doc / ADR-042 discussion): a real role transition replaces +/// `self.role` wholesale, which drops the old `FollowerState` — including +/// `pending_append_acks` and every sender inside it. This test drops the struct +/// directly rather than driving a full role-transition workflow, because that's +/// exactly what a role transition does to it; nothing here relies on any other part +/// of the transition machinery. +#[tokio::test] +async fn test_dropping_follower_state_releases_pending_ack_senders_as_closed() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let appended_index = 5u64; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: appended_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let append_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + + // Confirm the ACK is genuinely withheld (durable_index hasn't caught up) before + // dropping — otherwise this test wouldn't be exercising the pending-ack path at all. + assert!( + resp_rx.try_recv().is_err(), + "precondition: the ACK must still be withheld before the role transition" + ); + + // Simulates a real role transition: `self.role = self.role.become_xxx()?` drops + // the old FollowerState (and everything it owns) the same way this explicit + // drop does. + drop(state); + + // The withheld ACK's sender is gone — resp_rx must observe the channel closing, + // not hang forever and not receive a stale success response. + let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) + .await + .expect("recv() must resolve promptly once the sender is dropped, not hang"); + assert!( + result.is_err(), + "dropping FollowerState must close the pending ACK's channel, not deliver a \ + stale response" + ); +} + +/// #446: two independent AppendEntries requests (e.g. a leader retry) that both claim +/// the same threshold index must both eventually receive a response — the second one +/// landing on `pending_append_acks` must not silently overwrite the first. +#[tokio::test] +async fn test_multiple_requests_at_same_threshold_all_receive_response() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let claimed_index = 5u64; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .times(2) + .returning(move |_, _, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let append_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + // First request lands on threshold=5 and gets withheld. + let (resp_tx1, mut resp_rx1) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request.clone(), vec![resp_tx1]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + + // A second, independent request (e.g. leader retry) also claims index 5. + let (resp_tx2, mut resp_rx2) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request, vec![resp_tx2]), + &context, + internal_event_tx, + ) + .await + .unwrap(); + + assert!( + resp_rx1.try_recv().is_err(), + "first request must still be withheld" + ); + assert!( + resp_rx2.try_recv().is_err(), + "second request must still be withheld" + ); + + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(claimed_index, &context, &flush_tx).await; + + // Both senders — not just one — must receive a response. A single-value + // `BTreeMap` without `.entry().or_insert_with(...)` merging would + // let the second insert silently overwrite the first, dropping this ACK forever. + let response1 = resp_rx1.try_recv().expect("first sender must receive a response").unwrap(); + let response2 = resp_rx2 + .try_recv() + .expect("second sender must also receive a response, not be silently overwritten") + .unwrap(); + assert!(response1.is_success()); + assert!(response2.is_success()); +} + +/// #446: `LogFlushed` must release exactly the pending ACKs whose threshold is `<=` +/// durable_index — not `<` (off-by-one), and not all-or-nothing. +#[tokio::test] +async fn test_log_flushed_releases_only_thresholds_at_or_below_durable() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let call_count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .times(3) + .returning(move |_, _, _| { + let claimed = match call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) { + 0 => 10, + 1 => 12, + _ => 15, + }; + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: claimed, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let base_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let (tx10, mut rx10) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request.clone(), vec![tx10]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + let (tx12, mut rx12) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request.clone(), vec![tx12]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + let (tx15, mut rx15) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request, vec![tx15]), + &context, + internal_event_tx, + ) + .await + .unwrap(); + + assert!(rx10.try_recv().is_err()); + assert!(rx12.try_recv().is_err()); + assert!(rx15.try_recv().is_err()); + + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + + // durable_index advances to 12 — releases 10 and 12 (boundary is <=, not <), 15 stays. + state.handle_log_flushed(12, &context, &flush_tx).await; + assert!( + rx10.try_recv() + .expect("threshold 10 <= durable 12, must be released") + .unwrap() + .is_success() + ); + assert!( + rx12.try_recv() + .expect("threshold 12 <= durable 12 (boundary case), must be released") + .unwrap() + .is_success() + ); + assert!( + rx15.try_recv().is_err(), + "threshold 15 > durable 12, must still be withheld" + ); + + // durable_index advances to 15 — releases the rest. + state.handle_log_flushed(15, &context, &flush_tx).await; + assert!( + rx15.try_recv() + .expect("threshold 15 <= durable 15, must now be released") + .unwrap() + .is_success() + ); } /// Follower sends ACK immediately for heartbeat (no entries). @@ -3042,8 +3349,18 @@ async fn test_follower_commit_index_and_ack_both_sent_immediately() { new_commit, "commit_index must advance immediately" ); - let response = resp_rx.try_recv().expect("ACK must be sent immediately"); - assert!(response.unwrap().is_success()); + // RPO=0: commit_index advances immediately, but the ACK is withheld until durable. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); } /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 06b21cb8..f607bb06 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -1344,17 +1344,10 @@ impl RaftRoleState for LeaderState { internal_event_tx: &mpsc::UnboundedSender, ) { let new_commit_index = if self.cluster_metadata.single_voter { - // MemFirst single-voter: LogFlushed(durable) is the IO checkpoint. - // Commit to last_entry_id() — not just durable — to allow pipelining - // across IO batch boundaries. Matches multi-voter MemFirst where leader - // contributes last_entry_id() to quorum (not durable_index). - let last_log_index = ctx.raft_log().last_entry_id(); - debug_assert!( - last_log_index >= durable, - "last_entry_id ({last_log_index}) must be >= durable ({durable})" - ); - if last_log_index > self.commit_index() { - Some(last_log_index) + // RPO=0 (#446): single-voter has no majority to fall back on — commit + // must not advance past what this node has itself fsynced. + if durable > self.commit_index() { + Some(durable) } else { None } diff --git a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs index 3b9b8d07..2f6e7ec1 100644 --- a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs @@ -1,17 +1,16 @@ //! Single-Voter Commit Path Tests //! -//! Regression tests for the MemFirst single-voter commit path in `handle_log_flushed`. +//! RPO=0 (#446): `handle_log_flushed` single-voter branch must commit to `durable`, not +//! `last_entry_id()` — a single-voter cluster has no majority to fall back on, so if the +//! leader itself hasn't fsynced an entry, there is no copy anywhere safe from power loss. //! -//! ## Bug History -//! `fix #329` changed `handle_log_flushed` single-voter branch to commit to `durable` -//! instead of `last_entry_id()`. This placed IO thread latency on the commit critical -//! path, causing a ~3x latency regression in 3-node embedded bench (1731µs vs ~566µs). -//! -//! ## MemFirst Single-Voter Invariant -//! `LogFlushed(durable)` is an IO checkpoint. Commit must advance to `last_entry_id()` -//! — not just `durable` — to allow pipelining across IO batch boundaries. -//! This matches the multi-voter path where the leader contributes `last_entry_id()` to -//! quorum (not `durable_index`). +//! ## Superseded design (kept as history, do not resurrect) +//! `fix #329` changed this branch to commit to `durable` instead of `last_entry_id()`, +//! then reverted it after measuring a ~3x latency regression in 3-node embedded bench +//! (1731µs vs ~566µs) — IO thread latency landed on the commit critical path. That +//! regression is real and will resurface here. RPO=0 makes paying it mandatory for +//! single-voter clusters — there is no majority to absorb the risk the old design +//! accepted. use crate::MockMembership; use crate::MockRaftLog; @@ -61,16 +60,16 @@ async fn setup_single_voter( (state, ctx, last_entry_id) } -/// MemFirst single-voter: `handle_log_flushed` must commit to `last_entry_id`, not `durable`. +/// RPO=0: `handle_log_flushed` must commit to `durable`, not `last_entry_id`. /// -/// Simulates: IO batch flushed entries 1-3 (`durable=3`), but entries 4-5 arrived -/// in memory during the flush (`last_entry_id=5`). MemFirst: commit must advance -/// to 5 (all in-memory entries), not stall at 3 (only persisted entries). +/// Simulates: entries 4-5 arrived in memory (`last_entry_id=5`) but the IO batch has +/// only flushed entries 1-3 so far (`durable=3`). Commit must stay at 3 — entries 4-5 +/// aren't crash-safe yet, and a single-voter cluster has no other copy to fall back on. /// -/// This test FAILS if `handle_log_flushed` uses `durable` for commit -/// (the `fix #329` regression that caused +617µs avg latency in 3-node embedded bench). +/// This test FAILS if `handle_log_flushed` still uses `last_entry_id` for commit (the +/// old MemFirst behavior, since revoked — RPO=0 makes single-voter durability mandatory). #[tokio::test] -async fn test_single_voter_commit_uses_last_entry_id_not_durable() { +async fn test_single_voter_commit_uses_durable_not_last_entry_id() { // last_entry_id=5: entries 4-5 arrived in memory during the IO flush of 1-3 let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -80,13 +79,13 @@ async fn test_single_voter_commit_uses_last_entry_id_not_durable() { assert_eq!( state.commit_index(), - 5, - "MemFirst single-voter: commit must use last_entry_id=5, not durable=3. \ - Using durable puts IO latency on the commit critical path." + 3, + "RPO=0: commit must use durable=3, not last_entry_id=5 — entries 4-5 aren't \ + fsynced yet, and single-voter has no majority to fall back on" ); } -/// After IO catches up (durable == last_entry_id), commit equals last_entry_id. +/// After IO catches up (durable == last_entry_id), commit equals durable. #[tokio::test] async fn test_single_voter_commit_when_durable_equals_last_entry_id() { let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; @@ -94,20 +93,16 @@ async fn test_single_voter_commit_when_durable_equals_last_entry_id() { state.handle_log_flushed(5, &ctx, &internal_event_tx).await; - assert_eq!( - state.commit_index(), - 5, - "commit must advance to last_entry_id=5 when durable=5" - ); + assert_eq!(state.commit_index(), 5, "commit must advance to durable=5"); } -/// Pipelining across multiple IO batches: each flush triggers commit to current last_entry_id. +/// Commit tracks `durable` across IO batches, not the in-memory tail. /// -/// Simulates rapid writes where IO batches lag behind in-memory log: -/// - Flush 1: IO flushed 1-3, log has 1-7 in memory → commit=7 -/// - Flush 2: IO flushed 4-7, log has 1-10 in memory → commit=10 +/// Simulates rapid writes where the in-memory log runs ahead of what's fsynced: +/// - Flush 1: IO flushed 1-3 (durable=3), memory has 1-7 → commit=3, not 7 +/// - Flush 2: IO flushed 4-7 (durable=7), memory now has 1-10 → commit=7, not 10 #[tokio::test] -async fn test_single_voter_pipelining_across_io_batches() { +async fn test_single_voter_commit_tracks_durable_not_memory_tail() { let (mut state, ctx, last_entry_id) = setup_single_voter(7).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -115,8 +110,8 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 7, - "commit must advance to last_entry_id=7" + 3, + "commit must stay at durable=3 — entries 4-7 aren't fsynced yet" ); // IO batch 2: flushed 4-7, memory now has 1-10 @@ -124,14 +119,14 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(7, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 10, - "commit must advance to last_entry_id=10" + 7, + "commit must advance to durable=7, not the in-memory tail (10)" ); } -/// No-op flush: last_entry_id == commit_index means nothing new to commit. +/// No-op flush: durable == commit_index means nothing new is safe to commit yet. #[tokio::test] -async fn test_single_voter_no_commit_when_nothing_new() { +async fn test_single_voter_no_commit_when_nothing_new_durable() { let (mut state, ctx, _last_entry_id) = setup_single_voter(3).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -139,11 +134,11 @@ async fn test_single_voter_no_commit_when_nothing_new() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!(state.commit_index(), 3); - // Second flush with same last_entry_id=3: no new entries → no commit advance + // Second flush with same durable=3: nothing new is fsynced → no commit advance state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 3, - "commit must not advance when last_entry_id == commit_index" + "commit must not advance when durable == commit_index" ); } diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index 01c735b1..f465b3e8 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -22,6 +22,7 @@ use crate::alias::MOF; use crate::cluster_printer::print_learner_join_success; use crate::cluster_printer::print_learner_promoted_to_voter; use crate::cluster_printer::print_role_transition_line; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use async_trait::async_trait; use d_engine_proto::common::LogId; @@ -35,6 +36,7 @@ use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -88,6 +90,10 @@ pub struct LearnerState { /// reflected in the latest snapshot. pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -514,6 +520,10 @@ impl RaftRoleState for LearnerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl LearnerState { @@ -537,6 +547,7 @@ impl LearnerState { shared_state: SharedState::new(node_id, None, None), last_purged_index: None, snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), node_config, _marker: PhantomData, pending_purge_upto: None, @@ -646,6 +657,7 @@ impl From<&FollowerState> for LearnerState { shared_state: follower_state.shared_state.clone(), node_config: follower_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: follower_state.last_purged_index, pending_purge_upto: follower_state.pending_purge_upto, _marker: PhantomData, @@ -658,6 +670,7 @@ impl From<&CandidateState> for LearnerState { shared_state: candidate_state.shared_state.clone(), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, pending_purge_upto: None, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index 7eaf7bc7..ce68c796 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -363,6 +363,16 @@ async fn test_learner_handles_append_entries_success() { assert_eq!(state.current_term(), leader_term); assert_eq!(state.commit_index(), expected_commit); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + let response = resp_rx.recv().await.unwrap().unwrap(); assert!(response.is_success()); } @@ -1730,9 +1740,78 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { // MemFirst ACK Tests // ============================================================================ -/// Learner ACKs leader immediately after memory write (MemFirst). +/// Learner withholds the AppendEntries ACK until its own durable_index catches up. +/// +/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the +/// claimed index is fsynced. LogFlushed releases the withheld response. +#[tokio::test] +async fn test_learner_withholds_ack_until_durable() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let appended_index = 5u64; + + let mut replication_handler = crate::MockReplicationCore::new(); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + Ok(crate::AppendResponseWithUpdates { + response: d_engine_proto::server::replication::AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: appended_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = LearnerState::::new(1, context.node_config.clone()); + state.update_current_term(leader_term); + + let append_request = d_engine_proto::server::replication::AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + + // RPO=0: the ACK is withheld while durable_index < claimed index (5). + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); +} + +/// #446: if `LearnerState` is dropped (role transition — e.g. promotion to voter, or a +/// higher-term AppendEntries) while it still holds a withheld ACK, the pending sender +/// must be dropped with it — the caller waiting on `resp_rx` must see the channel +/// close, not hang forever and not receive a stale success. See the equivalent +/// Follower test for the full rationale — same mechanism, same reasoning. #[tokio::test] -async fn test_learner_acks_immediately_after_memory_write() { +async fn test_dropping_learner_state_releases_pending_ack_senders_as_closed() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); @@ -1778,9 +1857,22 @@ async fn test_learner_acks_immediately_after_memory_write() { .is_ok() ); - // MemFirst: ACK sent immediately - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); + assert!( + resp_rx.try_recv().is_err(), + "precondition: the ACK must still be withheld before the role transition" + ); + + // Simulates a real role transition dropping the old LearnerState. + drop(state); + + let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) + .await + .expect("recv() must resolve promptly once the sender is dropped, not hang"); + assert!( + result.is_err(), + "dropping LearnerState must close the pending ACK's channel, not deliver a \ + stale response" + ); } /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 89c6181e..f5948150 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -31,10 +31,13 @@ use d_engine_proto::common::LogId; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::replication::AppendEntriesRequest; use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::replication::SuccessResult; +use d_engine_proto::server::replication::append_entries_response; use d_engine_proto::server::storage::SnapshotAck; use d_engine_proto::server::storage::SnapshotChunk; use d_engine_proto::server::storage::SnapshotMetadata; use d_engine_proto::server::storage::SnapshotResponse; +use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::sync::mpsc; use tokio::time::Instant; @@ -56,6 +59,17 @@ pub(crate) enum PeerReplicationState { Snapshot, } +/// An AppendEntries response withheld because this node's own `durable_index` hasn't +/// caught up to what it would claim yet (RPO=0, #446). Keyed by the claimed index in +/// `pending_append_acks` (BTreeMap) — `senders` accumulates via +/// `entry().or_insert_with()` if more than one request lands on the same threshold +/// (retry, or a heartbeat landing on the same tail). +pub(crate) struct PendingAck { + pub(super) response: AppendEntriesResponse, + pub(super) senders: + Vec>>, +} + #[async_trait] pub(crate) trait RaftRoleState: Send + Sync + 'static { type T: TypeConfig; @@ -402,11 +416,25 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { /// Default: no-op for Candidate/Follower/Learner (ACK already sent on memory write). async fn handle_log_flushed( &mut self, - _durable: u64, + durable: u64, _ctx: &RaftContext, _internal_event_tx: &mpsc::UnboundedSender, ) { - // Candidate: no-op + // RPO=0 (#446): release any withheld AppendEntries responses whose claimed + // index is now durable. No-op for Candidate/Leader (pending_append_acks_mut + // returns None for them; Leader overrides this whole method anyway). + let Some(pending) = self.pending_append_acks_mut() else { + return; + }; + let later = pending.split_off(&(durable.saturating_add(1))); + let ready = std::mem::replace(pending, later); + for (_, ack) in ready { + for sender in ack.senders { + if let Err(e) = sender.send(Ok(ack.response)) { + error!("Failed to send released AppendEntriesResponse: {:?}", e); + } + } + } } /// Handle AppendEntries result from a per-follower ReplicationWorker. @@ -560,13 +588,48 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { } debug!("AppendEntriesResponse: {:?}", response); - // MemFirst: ACK immediately after memory write. IO thread fsyncs async. - // Safety: quorum uses last_entry_id (in-memory); crash safety is guaranteed by - // majority replication, not per-follower durability. + // RPO=0 (#446): a success response must not go out until this node's + // own durable_index has caught up to what it claims — an ACK asserts + // durability, and it must not lie about that. Conflict/higher-term + // responses don't claim any durable state, so they're never withheld. + let claimed_index = match &response.result { + Some(append_entries_response::Result::Success(SuccessResult { + last_match: Some(log_id), + })) => Some(log_id.index), + _ => None, + }; - for sender in senders { - if let Err(e) = sender.send(Ok(response)) { - error!("Failed to send: {:?}", e); + let withhold = + claimed_index.is_some_and(|idx| ctx.storage.raft_log.durable_index() < idx); + + if withhold { + let idx = claimed_index.unwrap(); + match self.pending_append_acks_mut() { + Some(pending) => { + pending + .entry(idx) + .or_insert_with(|| PendingAck { + response, + senders: Vec::new(), + }) + .senders + .extend(senders); + } + None => { + // Should never happen — only Follower/Learner reach this + // branch. Fail loud rather than silently dropping an ACK + // the leader is waiting on. + error!("no pending_append_acks slot on a role that should have one"); + for sender in senders { + let _ = sender.send(Ok(response)); + } + } + } + } else { + for sender in senders { + if let Err(e) = sender.send(Ok(response)) { + error!("Failed to send: {:?}", e); + } } } } @@ -940,6 +1003,12 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { _state: PeerReplicationState, ) { } + + /// Follower/Learner's withheld-response queue. `None` for Candidate/Leader — + /// same pattern as `pending_purge_upto_mut` below. + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + None + } } /// Attempts to execute whatever purge target is currently pending, if any. Shared by both diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index b841238e..0fb7960d 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -359,6 +359,9 @@ where } } + // #446: this is what election-eligibility comparisons (is_target_log_more_recent) + // read. It must keep reflecting the in-memory tail, never durable_index — a node + // with an un-fsynced entry must still be able to reject a less-up-to-date candidate. fn last_log_id(&self) -> Option { let last_index = self.last_entry_id(); if last_index > 0 { @@ -640,11 +643,10 @@ where mut peer_matched_ids: Vec, ) -> Option { let _timer = ScopedTimer::new("calculate_majority_matched_index"); - // Leader's contribution: last_entry_id (in-memory). With MemFirst (Level 2), db.write() - // returns once data reaches OS page cache — durable_index advances immediately. - // Followers also ACK after OS page cache write (no fsync wait). Crash safety is - // OS page cache level: process crash is recoverable, power loss is not. - peer_matched_ids.push(self.last_entry_id()); + // RPO=0 (#446): leader's own contribution must be its own durable (fsynced) + // position, not the in-memory tail — otherwise a majority-looking commit can + // still lose data on correlated power loss. + peer_matched_ids.push(self.durable_index()); // Sort in descending order peer_matched_ids.sort_unstable_by(|a, b| b.cmp(a)); @@ -775,8 +777,8 @@ where idle_flush_interval_ms, } = persistence_config.flush_policy; debug!( - "Creating BufferedRaftLog with node_id: {}, strategy: {:?}, idle_flush_interval_ms: {:?}, disk_len: {:?}", - node_id, persistence_config.strategy, idle_flush_interval_ms, disk_len + "Creating BufferedRaftLog with node_id: {}, idle_flush_interval_ms: {:?}, disk_len: {:?}", + node_id, idle_flush_interval_ms, disk_len ); let shutdown_timeout_ms = persistence_config.shutdown_timeout_ms; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs index 6790f539..cb12c6d7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs @@ -13,7 +13,7 @@ use crate::test_utils::{ BufferedRaftLogTestContext, mock_empty_entries, simulate_delete_command, simulate_insert_command, }; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test get_entries_range returns correct subset /// @@ -24,7 +24,6 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; #[tokio::test] async fn test_get_entries_range_returns_correct_subset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,7 +51,6 @@ async fn test_get_entries_range_returns_correct_subset() { #[tokio::test] async fn test_get_entries_range_handles_large_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -86,7 +84,6 @@ async fn test_get_entries_range_handles_large_range() { #[tokio::test] async fn test_filter_conflicts_removes_entries_with_different_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -133,7 +130,6 @@ async fn test_filter_conflicts_removes_entries_with_different_term() { #[tokio::test] async fn test_filter_conflicts_handles_multiple_scenarios() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -193,7 +189,6 @@ async fn test_filter_conflicts_handles_multiple_scenarios() { #[tokio::test] async fn test_last_entry_returns_highest_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -218,7 +213,6 @@ async fn test_last_entry_returns_highest_index() { #[tokio::test] async fn test_last_entry_matches_buffer_length() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -247,7 +241,6 @@ async fn test_last_entry_matches_buffer_length() { #[tokio::test] async fn test_last_entry_with_large_payload_id() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -272,7 +265,6 @@ async fn test_last_entry_with_large_payload_id() { #[tokio::test] async fn test_insert_batch_appends_entries_in_order() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -305,7 +297,6 @@ async fn test_insert_batch_appends_entries_in_order() { #[tokio::test] async fn test_get_entries_range_multiple_bounds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -360,7 +351,6 @@ async fn test_get_entries_range_multiple_bounds() { #[tokio::test] async fn test_insert_duplicate_commands_as_separate_events() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -395,7 +385,6 @@ async fn test_insert_duplicate_commands_as_separate_events() { #[tokio::test] async fn test_purge_after_insert_maintains_consistency() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -428,7 +417,6 @@ async fn test_purge_after_insert_maintains_consistency() { #[tokio::test] async fn test_purge_logs_removes_entries_up_to_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -476,7 +464,6 @@ async fn test_purge_logs_removes_entries_up_to_index() { #[tokio::test] async fn test_concurrent_purge_operations_are_safe() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -515,7 +502,6 @@ async fn test_concurrent_purge_operations_are_safe() { #[tokio::test] async fn test_first_entry_id_after_purge_updates() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -547,7 +533,6 @@ async fn test_first_entry_id_after_purge_updates() { #[tokio::test] async fn test_single_entry_insert_succeeds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -569,7 +554,6 @@ async fn test_single_entry_insert_succeeds() { #[tokio::test] async fn test_is_empty_returns_true_for_new_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -588,7 +572,6 @@ async fn test_is_empty_returns_true_for_new_log() { #[tokio::test] async fn test_is_empty_returns_false_after_append() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -613,7 +596,6 @@ async fn test_is_empty_returns_false_after_append() { #[tokio::test] async fn test_last_log_id_for_empty_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -636,7 +618,6 @@ async fn test_last_log_id_for_empty_log() { #[tokio::test] async fn test_last_log_id_after_appends() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -662,7 +643,6 @@ async fn test_last_log_id_after_appends() { #[tokio::test] async fn test_drop_shuts_down_workers_gracefully() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -686,14 +666,12 @@ async fn test_drop_shuts_down_workers_gracefully() { #[tokio::test] async fn test_same_index_and_term_implies_identical_prefix() { let ctx1 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, "test_log_matching_1", ); let ctx2 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -726,7 +704,6 @@ async fn test_same_index_and_term_implies_identical_prefix() { #[tokio::test] async fn test_committed_entry_present_in_future_leaders() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -758,7 +735,6 @@ async fn test_committed_entry_present_in_future_leaders() { #[tokio::test] async fn test_append_updates_last_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -784,7 +760,6 @@ async fn test_append_updates_last_entry() { #[tokio::test] async fn test_insert_batch_with_empty_list() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -807,7 +782,6 @@ async fn test_insert_batch_with_empty_list() { #[tokio::test] async fn test_insert_batch_updates_metadata() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index fce20bf3..8cde96b9 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -8,7 +8,7 @@ use crate::{ BufferedRaftLog, FlushPolicy, InternalEvent, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; use std::sync::Arc; @@ -34,7 +34,6 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -85,34 +84,35 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { ); } -/// `calculate_majority_matched_index` uses the in-memory SkipMap (`last_entry_id`), -/// not `durable_index` — even when all fsyncs are stalled indefinitely. +/// `calculate_majority_matched_index` uses `durable_index` (fsync-confirmed), not the +/// in-memory `last_entry_id` — even when a follower already reports the index, the +/// leader's own contribution must not count toward quorum until it has itself fsynced. /// -/// Stall every flush() call via a MockLogStore barrier, append entries, then verify -/// that majority-matched calculation returns the correct in-memory index. +/// Stall every flush() call via a MockLogStore barrier, append entries, then verify that +/// majority-matched calculation does NOT advance while fsync is stalled, and does advance +/// once fsync completes. /// -/// Regression guard: if majority calculation ever changes to depend on `durable_index`, -/// this test will catch it before it reaches production. +/// Regression guard: RPO=0 (#446) requires the leader's own copy to be durable before it +/// counts toward commit — if this ever reverts to using `last_entry_id`, this test will +/// catch it before it reaches production. /// /// Expected: -/// - Append entries so `last_entry_id()` reaches N (e.g. 5) while fsync is -/// permanently stalled — `durable_index()` stays at its pre-write value -/// (0) throughout. -/// - Feed `calculate_majority_matched_index` a `match_index` map where enough -/// followers already report N to form a majority. -/// - Assert the returned majority-matched index equals N (matching -/// `last_entry_id()`) — NOT 0 (what it would return if it mistakenly used -/// `durable_index()` instead). +/// - Append entries so `last_entry_id()` reaches N (e.g. 2) while fsync is permanently +/// stalled — `durable_index()` stays at its pre-write value (0) throughout. +/// - Feed `calculate_majority_matched_index` a `match_index` map where one follower +/// already reports N=2 (majority IF the leader's own un-fsynced entry counted) — +/// assert the result is `None` while durable_index is still 0. +/// - Release the gate. Once `durable_index` reaches 2, the same call must return +/// `Some(2)`. #[tokio::test] -async fn test_majority_matched_index_uses_memory_not_durable_index() { +async fn test_majority_matched_index_uses_durable_not_memory() { // Gate closed: the first flush() call will block until we send () on `flush_gate`. let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( - "majority_matched_index_uses_memory_not_durable_index".into(), + "majority_matched_index_uses_durable_not_memory".into(), ); let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify should trigger fsync here. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -160,22 +160,22 @@ async fn test_majority_matched_index_uses_memory_not_durable_index() { assert_eq!( raft_log.last_entry_id(), pre_last_entry_id + size, - "durable_index must not advance before fsync completes" + "in-memory tail should still advance even while fsync is stalled" ); // One follower already matched index 2; the other is still behind at 0 — asymmetric - // on purpose. With only ONE follower at 2, the leader's own contribution decides - // whether the majority (2 out of 3 voters) reaches 2. If this ever regresses to use - // `durable_index()` (0, since fsync is still gated) instead of `last_entry_id()` (2), - // the median drops to 0 and the call returns `None` instead of `Some(2)`. + // on purpose. If the leader's own un-fsynced entry counted (the old MemFirst + // behavior), 2 out of 3 voters would reach index 2 — but RPO=0 requires the + // leader's own copy to be durable first, so this must return None while fsync + // is still gated. let result = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); assert_eq!( - result, - Some(2), - "majority index must use last_entry_id (2), not durable_index (0)" + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even \ + when a follower already reports it" ); - // Release the gate — flush() returns, advance_durable_and_notify(1) fires. + // Release the gate — flush() returns, advance_durable_and_notify(2) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, @@ -187,6 +187,14 @@ async fn test_majority_matched_index_uses_memory_not_durable_index() { pre_write_durable_index + size, "durable_index must reach the expected index after fsync completes" ); + + // Now the leader's own contribution is durable (2), so the same call must succeed. + let result_after_fsync = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); + assert_eq!( + result_after_fsync, + Some(2), + "once the leader's own entry is durable, majority index must advance to 2" + ); } /// `entry_term()` returns the correct term during high-concurrency writes @@ -212,7 +220,6 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -300,7 +307,6 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -368,7 +374,6 @@ async fn test_flush_caller_blocked_until_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -442,7 +447,6 @@ async fn test_flush_callers_arriving_during_inflight_fsync_are_coalesced() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -526,7 +530,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_ok_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -600,7 +603,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_err_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -681,7 +683,6 @@ async fn test_reset_during_inflight_fsync_does_not_resurrect_stale_durable_index let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -763,7 +764,6 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs index 2b345e79..d3201d2f 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs @@ -3,15 +3,14 @@ use std::time::Duration; use futures::future::join_all; use tokio; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_remove_range_with_concurrent_reads() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,7 +53,6 @@ async fn test_remove_range_with_concurrent_reads() { #[tokio::test] async fn test_concurrent_append_and_purge() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -128,7 +126,6 @@ async fn test_get_entries_range_never_returns_torn_result_during_concurrent_purg const ITERATIONS: usize = 500; let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index daf9a096..bf4e3996 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -20,7 +20,6 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; use d_engine_proto::common::Entry; use d_engine_proto::common::LogId; use std::sync::Arc; @@ -156,7 +155,6 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -288,7 +286,6 @@ async fn test_flush_propagates_io_error() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -350,7 +347,6 @@ async fn test_fsync_failure_poisons_and_rejects_writes_after_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -407,7 +403,6 @@ async fn test_replace_range_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -497,7 +492,6 @@ async fn test_purge_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -547,7 +541,6 @@ async fn test_reset_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -596,7 +589,6 @@ async fn test_save_hard_state_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -645,7 +637,6 @@ async fn test_poisoned_rejects_save_hard_state() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -693,7 +684,6 @@ async fn test_poisoned_skips_replace_range() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -775,7 +765,6 @@ async fn test_poisoned_does_not_skip_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -816,7 +805,6 @@ async fn test_poisoned_skips_purge() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -899,7 +887,6 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1010,7 +997,6 @@ async fn test_new_buffered_raft_log_starts_unpoisoned() { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -1042,7 +1028,6 @@ async fn test_poisoned_survives_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -1082,7 +1067,6 @@ async fn test_persist_entries_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1149,7 +1133,6 @@ async fn test_poisoned_rejects_queued_persist_task() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1216,7 +1199,6 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs index 521338b3..2c4f0c02 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs @@ -6,16 +6,12 @@ use tokio::sync::mpsc; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, MockStorageEngine, simulate_insert_command}; -use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_durable_index_monotonic_under_concurrency() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -55,7 +51,6 @@ async fn test_durable_index_monotonic_under_concurrency() { #[tokio::test] async fn test_durable_index_with_non_contiguous_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,7 +121,6 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs index 60f00876..15ff8451 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs @@ -1,14 +1,13 @@ use bytes::Bytes; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, EntryPayload, LogId}; #[tokio::test] async fn test_empty_log_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -27,7 +26,6 @@ async fn test_empty_log_operations() { #[tokio::test] async fn test_single_entry_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -56,7 +54,6 @@ async fn test_single_entry_operations() { #[tokio::test] async fn test_gap_handling_in_indexes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -99,7 +96,6 @@ async fn test_gap_handling_in_indexes() { #[tokio::test] async fn test_extreme_boundary_conditions() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs index 44fd0040..95e40d8b 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs @@ -13,7 +13,7 @@ use d_engine_proto::common::{Entry, EntryPayload}; use tokio::time::{Duration, sleep}; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test MemFirst with threshold=1 persists entries after flush /// @@ -23,7 +23,6 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; #[tokio::test] async fn test_mem_first_entries_durable_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,7 +51,6 @@ async fn test_mem_first_entries_durable_after_flush() { #[tokio::test] async fn test_mem_first_concurrent_writes_durable_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -102,7 +100,6 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { #[tokio::test] async fn test_mem_first_crash_recovery_restores_flushed_entries() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -149,7 +146,6 @@ async fn test_mem_first_crash_recovery_restores_flushed_entries() { #[tokio::test] async fn test_mem_first_buffers_entries_before_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -173,7 +169,6 @@ async fn test_mem_first_buffers_entries_before_flush() { #[tokio::test] async fn test_mem_first_flushes_asynchronously() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -202,7 +197,6 @@ async fn test_mem_first_flushes_asynchronously() { #[tokio::test] async fn test_mem_first_concurrent_buffering() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -243,7 +237,6 @@ async fn test_mem_first_concurrent_buffering() { #[tokio::test] async fn test_batched_flushes_at_threshold() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10000, // High interval to test threshold trigger }, @@ -269,7 +262,6 @@ async fn test_batched_flushes_at_threshold() { #[tokio::test] async fn test_batched_flushes_at_interval() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -295,7 +287,6 @@ async fn test_batched_flushes_at_interval() { #[tokio::test] async fn test_batched_partial_flush_recovery() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs index 095db56d..508c3bd5 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs @@ -11,8 +11,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; fn setup_memory() -> Arc> { @@ -20,7 +19,6 @@ fn setup_memory() -> Arc> { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs index a9530bf9..52826bfb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs @@ -12,7 +12,7 @@ use tokio::time::Instant; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,24 +47,17 @@ async fn test_reset_performance_during_active_flush() { let max_reset_duration_ms = FLUSH_DELAY_MS * 3; // 600ms: accounts for IO thread overhead let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -100,9 +93,8 @@ async fn test_reset_performance_during_active_flush() { assert!( duration.as_millis() < max_reset_duration_ms as u128, - "Reset took {}ms during active flush ({:?}/{:?})", + "Reset took {}ms during active flush ({:?})", duration.as_millis(), - strategy, flush_policy ); } @@ -124,7 +116,6 @@ async fn test_filter_conflicts_performance_during_flush() { for (idle_flush_interval_ms, max_duration_ms) in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -190,21 +181,15 @@ async fn test_fresh_cluster_performance_consistency() { let max_duration_ms = if is_ci { 50 } else { 5 }; let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let mut log_store = MockLogStore::new(); log_store.expect_is_write_durable().returning(|| true); log_store.expect_flush().return_once(|| Ok(())); @@ -215,7 +200,6 @@ async fn test_fresh_cluster_performance_consistency() { log_store.expect_reset().returning(|| Ok(())); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -235,9 +219,8 @@ async fn test_fresh_cluster_performance_consistency() { assert!( duration.as_millis() < max_duration_ms as u128, - "Fresh cluster reset took {}ms ({:?}/{:?})", + "Fresh cluster reset took {}ms ({:?})", duration.as_millis(), - strategy, flush_policy ); } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs index e91d3cb5..95f49106 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -28,10 +28,7 @@ use d_engine_proto::common::Entry; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; fn entry( index: u64, @@ -49,7 +46,6 @@ fn entry( #[tokio::test] async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // isolate from the safety-net timer }, @@ -134,7 +130,6 @@ async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index efb54b80..a0d689fa 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -13,12 +13,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -472,7 +471,6 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs index 3819e101..1a206f84 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -20,7 +20,7 @@ use d_engine_proto::common::Entry; use crate::{ BufferedRaftLog, FlushPolicy, LogStore, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, StorageEngine, + RaftLog, StorageEngine, }; /// `append_entries()` must not return before the entry reaches the storage @@ -48,7 +48,6 @@ async fn test_append_entries_waits_for_storage_engine_before_returning() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs index 0429eebc..36b88ce3 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs @@ -1,62 +1,194 @@ //! Quorum Durability Tests //! -//! MemFirst design: leader contributes `last_entry_id` (in-memory) to quorum. -//! IO thread persistence is async and NOT on the commit critical path. +//! RPO=0 (#446): leader contributes `durable_index` (not `last_entry_id`) to quorum — +//! commit must not advance past what the leader itself has survived fsync for. //! -//! Follower ACK path: followers ACK immediately after memory write (no `wait_durable`). -//! IO thread fsyncs asynchronously; crash safety is guaranteed by quorum, not per-follower durability. +//! Superseded design (kept here as history, do not resurrect): the old MemFirst model had +//! the leader contribute `last_entry_id` (in-memory) so IO persistence never sat on the +//! commit critical path. That traded away RPO=0 — a majority-acked write could still be +//! lost on correlated power loss before fsync. This file's tests now lock in the new +//! behavior instead of the old one. +//! +//! Follower ACK path (tracked separately, not yet landed): followers will ACK only after +//! their own durable_index catches up — so a follower's reported match_index is inherently +//! already durable by the time the leader sees it. +//! +//! Election-eligibility comparison must keep reading the in-memory log, never +//! `durable_index` — a separate, independent invariant from the durable-quorum change +//! above, but one a majority-count safety argument for #446 depends on. See +//! `test_election_eligibility_reads_memory_log_not_durable_index`. +//! +//! Note: these tests rely on `BufferedRaftLog`'s in-memory layer existing (they force a +//! gap between `last_entry_id` and `durable_index` via a gated mock flush). If that layer +//! is ever removed, this file's setup assumptions need revisiting — not a decided plan, +//! just a known dependency to check first. +//! +//! Tests that need a genuine, un-fsynced gap between `last_entry_id` and `durable_index` +//! use `MockStorageEngine::not_durable_gated_flush` — a real channel-based gate, not a +//! timing guess. An earlier version of this file relied on a long `idle_flush_interval_ms` +//! and assumed the dedicated `raft-io-*` OS thread just wouldn't get scheduled before the +//! assertions ran; that's a real race (the IO thread is independent of the test's own +//! runtime), and it was intermittently losing under load — flaky, not broken logic. Do not +//! reintroduce that pattern here. +use crate::BufferedRaftLog; +use crate::FlushPolicy; +use crate::MockStorageEngine; +use crate::MockTypeConfig; +use crate::PersistenceConfig; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; +use std::sync::Arc; use std::time::Duration; -/// Flush policy with a far-future safety timer — IO thread only fsyncs on WriteNotify. -/// In current_thread test runtime, durable_index stays at 0 immediately after append_entries -/// because the IO thread task has no chance to run until the test yields. -fn no_auto_flush_policy() -> FlushPolicy { - FlushPolicy::Batch { - idle_flush_interval_ms: 999_999, - } +/// Entries `1..=n`, all at `term`, no payload — the shape these tests need. +fn entries( + n: u64, + term: u64, +) -> Vec { + (1..=n) + .map(|index| Entry { + index, + term, + payload: None, + }) + .collect() } -// ── Leader quorum uses last_entry_id (in-memory), not durable_index ── +// ── Leader quorum uses durable_index, not last_entry_id (RPO=0) ── -/// MemFirst: leader's quorum contribution is last_entry_id (in-memory), not durable_index. +/// RPO=0: leader's quorum contribution is durable_index (fsync-confirmed), not +/// last_entry_id (in-memory). /// -/// Even when durable_index=0 (IO thread has not flushed), quorum must be satisfied -/// as soon as last_entry_id + follower ACKs form a majority. IO persistence is async -/// and must NOT block commit. +/// Even when a follower has already ACKed an index, the leader must not count its own +/// un-fsynced entry toward quorum — otherwise a majority-looking commit can still lose +/// data on correlated power loss (the leader's own copy was never actually durable). /// -/// This test FAILS if calculate_majority_matched_index uses durable_index (the bug -/// introduced by fix #329 which incorrectly put IO thread latency on the commit -/// critical path, causing +617µs avg latency regression in 3-node embedded bench). +/// This test FAILS if calculate_majority_matched_index still uses last_entry_id (the old +/// MemFirst behavior, since revoked). It replaces +/// `test_memfirst_quorum_uses_last_entry_id_not_durable_index`, which asserted the exact +/// opposite of this on purpose — that assertion documented a since-revoked design decision. #[tokio::test] -async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), // durable_index stays 0 — IO thread won't run - "test_memfirst_quorum_last_entry_id", +async fn test_quorum_uses_durable_index_not_last_entry_id() { + // Gate closed: the first flush() call blocks until we send () on `flush_gate` — fsync + // deterministically never completes until we say so, no timing involved. + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_quorum_durable_index".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Entry written to SkipMap (in memory). IO thread has not flushed yet. - ctx.append_entries(1, 1, 1).await; + raft_log.append_entries(entries(1, 1)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; // let it reach the gate - assert_eq!(ctx.raft_log.last_entry_id(), 1); - assert_eq!(ctx.raft_log.durable_index(), 0); // IO thread hasn't run + assert_eq!(raft_log.last_entry_id(), 1); + assert_eq!(raft_log.durable_index(), 0); // gate never released — fsync hasn't completed - let result = ctx.raft_log.calculate_majority_matched_index( + let result = raft_log.calculate_majority_matched_index( 1, 0, - vec![1], // one follower acked index=1; together with leader = majority of 3 + vec![1], // one follower reports match=1 (already durable, post-Stage2 semantics) + ); + + // RPO=0: leader contributes durable_index=0, not last_entry_id=1. + // peer_matched_ids = [follower=1, leader=0], sorted desc = [1,0], median(len/2=1) = 0. + // majority_index=0 is not < commit_index=0, so falls through to the term check on + // entry(0) — index 0 is not a real entry (log is 1-indexed) — Ok(None) — result is None. + assert_eq!( + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even when \ + a follower has already acked it — one follower alone isn't majority without the \ + leader's own durable contribution" + ); + + let _ = flush_gate.send(()); // release so the blocked IO thread doesn't linger +} + +/// Election-eligibility comparison (`last_log_id`, consumed by +/// `election_handler::handle_vote_request`) must read the in-memory log, never +/// `durable_index`. A follower with an un-fsynced tail must still be able to correctly +/// reject a candidate whose log is genuinely less up to date — voting eligibility and +/// commit-durability are two separate concerns and must not be conflated by sharing the +/// same index source. +#[tokio::test] +async fn test_election_eligibility_reads_memory_log_not_durable_index() { + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_election_eligibility_memory_log".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + raft_log.append_entries(entries(10, 1)).await.unwrap(); // entries 1..=10, term=1 + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(raft_log.durable_index(), 0, "nothing fsynced yet"); + assert_eq!( + raft_log.last_log_id(), + Some(LogId { index: 10, term: 1 }), + "election-eligibility comparison must see the un-fsynced tail, not fall back to \ + durable_index=0 — a candidate with a truly-shorter log must still be rejected" + ); + + let _ = flush_gate.send(()); +} + +/// `calculate_majority_matched_index`'s median-based calculation requires an actual +/// majority of `peer_matched_ids` to reach an index before it counts toward commit — a +/// minority (here: 2 of 5) reporting a higher index cannot move the result past what the +/// rest of the cluster last confirmed. +/// +/// This is a pure property of the median calculation itself — the function has no way to +/// know whether any of its inputs are stale or expired, so this test does not by itself +/// prove anything about stale reports being harmless. It only pins down the arithmetic +/// that a separate, broader safety argument for #446 relies on. +#[tokio::test] +async fn test_majority_matched_index_requires_actual_majority_of_reports() { + let ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_majority_requires_actual_majority", ); - // MemFirst: leader contributes last_entry_id=1. - // quorum = [leader=1, follower=1] → majority of {leader, f1, f2} satisfied → Some(1). + ctx.append_entries(1, 10, 1).await; + ctx.raft_log.flush().await.unwrap(); // leader's own entries now durable through 10 + + assert_eq!(ctx.raft_log.durable_index(), 10); + + // 1 of 4 followers reports index 10; the other 3 are still at their last-known + // value, 9. + let result = ctx.raft_log.calculate_majority_matched_index(1, 9, vec![10, 9, 9, 9]); + + // peer_matched_ids after leader's own contribution = [10, 9, 9, 9, 10] + // sorted desc = [10,10,9,9,9], median(len/2=2) = 9 — majority stays at 9, entry(9) + // exists with term=1=current_term, so the result is the previously-safe Some(9), not 10. assert_eq!( result, - Some(1), - "MemFirst: quorum must use last_entry_id, not durable_index — IO must not block commit" + Some(9), + "a minority (2 of 5) reporting a higher index cannot move majority past what the \ + other 3 nodes last confirmed" ); } @@ -67,7 +199,6 @@ async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { #[tokio::test] async fn test_quorum_succeeds_after_leader_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 999_999, // only threshold trigger, no timer }, @@ -103,35 +234,47 @@ async fn test_quorum_succeeds_after_leader_flush() { // ── Bug 2: gap between last_entry_id and durable_index ── -/// Demonstrates that after append_entries with MemFirst + no-auto-flush, -/// last_entry_id and durable_index diverge. +/// Demonstrates that after append_entries with a stalled fsync, last_entry_id and +/// durable_index diverge. /// /// This is the root condition enabling the bug: both values exist, /// but quorum calculation only uses the unsafe one. #[tokio::test] async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), - "test_diverge_mem_first", + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_diverge_mem_first".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); - ctx.append_entries(1, 5, 1).await; // entries 1..=5, no flush + raft_log.append_entries(entries(5, 1)).await.unwrap(); // entries 1..=5, no flush + tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(ctx.raft_log.last_entry_id(), 5, "memory index should be 5"); + assert_eq!(raft_log.last_entry_id(), 5, "memory index should be 5"); assert_eq!( - ctx.raft_log.durable_index(), + raft_log.durable_index(), 0, "durable_index must remain 0: no flush has run" ); // This gap (5 vs 0) is exactly what the quorum bug exploits. + + let _ = flush_gate.send(()); } /// After explicit flush, durable_index must equal last_entry_id. #[tokio::test] async fn test_durable_index_equals_last_entry_id_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs index f7acfe7f..373aeffd 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs @@ -1,12 +1,11 @@ +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::Entry; #[tokio::test] async fn test_log_matching_property() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -46,7 +45,6 @@ async fn test_log_matching_property() { #[tokio::test] async fn test_leader_completeness_property() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -84,7 +82,6 @@ async fn test_leader_completeness_property() { #[tokio::test] async fn test_calculate_majority_matched_index_case0() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -108,7 +105,6 @@ async fn test_calculate_majority_matched_index_case0() { #[tokio::test] async fn test_calculate_majority_matched_index_case1() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -132,7 +128,6 @@ async fn test_calculate_majority_matched_index_case1() { #[tokio::test] async fn test_calculate_majority_matched_index_case2() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -157,7 +152,6 @@ async fn test_calculate_majority_matched_index_case2() { #[tokio::test] async fn test_calculate_majority_matched_index_case3() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -180,7 +174,6 @@ async fn test_calculate_majority_matched_index_case3() { #[tokio::test] async fn test_calculate_majority_matched_index_case4() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -204,7 +197,6 @@ async fn test_calculate_majority_matched_index_case4() { #[tokio::test] async fn test_calculate_majority_matched_index_case5() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs index 656e2c25..c62dc1eb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs @@ -1,8 +1,8 @@ use d_engine_proto::common::{Entry, LogId}; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; fn entry( index: u64, @@ -18,7 +18,6 @@ fn entry( #[tokio::test] async fn test_remove_middle_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -50,7 +49,6 @@ async fn test_remove_middle_range() { #[tokio::test] async fn test_remove_from_start() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -77,7 +75,6 @@ async fn test_remove_from_start() { #[tokio::test] async fn test_remove_to_end() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -105,7 +102,6 @@ async fn test_remove_to_end() { #[tokio::test] async fn test_remove_empty_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,7 +122,6 @@ async fn test_remove_empty_range() { #[tokio::test] async fn test_remove_entire_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -150,7 +145,6 @@ async fn test_remove_entire_log() { #[tokio::test] async fn test_remove_single_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -185,7 +179,6 @@ async fn test_remove_single_entry() { #[tokio::test] async fn test_remove_range_clears_term_indexes_for_removed_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -252,7 +245,6 @@ async fn test_remove_range_clears_term_indexes_for_removed_entries() { #[tokio::test] async fn test_purge_prefix_removes_entries_and_records_boundary_together() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -301,7 +293,6 @@ async fn test_purge_prefix_removes_entries_and_records_boundary_together() { #[tokio::test] async fn test_purge_prefix_multi_term_cutoff_updates_term_indexes_and_boundary() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs index 4f0beb2d..5f0358c7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -17,9 +17,9 @@ use std::time::Duration; use d_engine_proto::common::Entry; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; fn entry( index: u64, @@ -45,7 +45,6 @@ fn entry( #[tokio::test] async fn test_replace_range_becomes_durable_without_a_following_append() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs index c14d7c4a..ad909049 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs @@ -7,7 +7,7 @@ use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, + PersistenceConfig, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,7 +47,6 @@ fn test_io_thread_survives_runtime_drop() { let (log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -84,7 +83,6 @@ fn test_io_thread_survives_runtime_drop() { #[tokio::test] async fn test_shutdown_closes_channel_properly() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -113,7 +111,6 @@ async fn test_shutdown_closes_channel_properly() { #[tokio::test] async fn test_shutdown_awaits_worker_completion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -159,7 +156,6 @@ async fn test_shutdown_handles_slow_workers() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -203,7 +199,6 @@ async fn test_shutdown_handles_slow_workers() { #[tokio::test] async fn test_shutdown_with_multiple_flushes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -289,7 +284,6 @@ async fn test_replace_range_failure_propagates_error_and_shuts_down_io_thread() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // no auto-flush }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs index 5e37ef0d..cec818e4 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs @@ -8,12 +8,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; #[tokio::test] async fn test_first_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -88,7 +87,6 @@ async fn test_first_index_for_term() { #[tokio::test] async fn test_last_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -162,7 +160,6 @@ async fn test_last_index_for_term() { #[tokio::test] async fn test_term_index_functions_with_purged_logs() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -214,7 +211,6 @@ async fn test_term_index_functions_with_purged_logs() { #[tokio::test] async fn test_term_index_sequential_multi_term_insertion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -263,7 +259,6 @@ async fn test_term_index_sequential_multi_term_insertion() { #[tokio::test] async fn test_term_indexes_rebuilt_correctly_after_restart() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -304,7 +299,6 @@ async fn test_term_indexes_rebuilt_correctly_after_restart() { #[tokio::test] async fn test_term_index_performance_large_dataset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs index a385cd5d..5f36a0f6 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs @@ -16,7 +16,7 @@ use d_engine_proto::common::Entry; use crate::storage::buffered_raft_log::TermSegments; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; // --------------------------------------------------------------------------- // Helpers @@ -37,7 +37,6 @@ fn entries( fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs index c2532224..16714b50 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -21,10 +21,7 @@ use std::time::Duration; use d_engine_proto::common::Entry; use crate::storage::raft_log::RaftLog; -use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; fn entry( index: u64, @@ -54,7 +51,6 @@ async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs index c2181802..965e8cf2 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs @@ -1,15 +1,14 @@ use std::time::Duration; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; /// Verifies that the flush worker continues operating normally after processing a large number /// of flush tasks — the worker does not exit or become unresponsive under sustained load. #[tokio::test] async fn test_flush_worker_sustains_throughput_under_load() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 32f21ffd..3b2a371e 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -19,7 +19,6 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; use crate::Result; use std::sync::Arc; @@ -31,7 +30,6 @@ fn minimal_raft_log(storage: MockStorageEngine) -> Arc::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index b21778bc..0b9b1496 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -209,14 +209,14 @@ pub trait RaftLog: Send + Sync + 'static { /// - Persist entries to durable storage BEFORE updating in-memory state /// - Call fsync/flush before returning Ok(()) /// - Ensures entries survive crashes immediately - /// - Example: BufferedRaftLog with PersistenceStrategy::DiskFirst + /// - Example: a store that fsyncs before returning /// /// 2. **Memory-First (Performance-optimized, Acceptable for Followers)**: /// - Update in-memory state first /// - Enqueue entries for asynchronous durability /// - MUST guarantee eventual durability via background flush /// - MUST call flush() before acknowledging commits - /// - Example: BufferedRaftLog with PersistenceStrategy::MemFirst + /// - Example: a store that fsyncs asynchronously /// - WARNING: Leader MUST wait_durable() before responding to AppendEntries RPCs /// /// # Safety Invariants @@ -228,10 +228,10 @@ pub trait RaftLog: Send + Sync + 'static { /// - MUST update term indexes (first/last_index_for_term) atomically /// /// # Raft Protocol Integration - /// - Leaders using MemFirst MUST call wait_durable(index) before: + /// - Leaders using async fsync MUST call wait_durable(index) before: /// * Responding success to AppendEntries RPC /// * Advancing commit index - /// - Followers can use MemFirst safely because leader durability guarantees safety + /// - Followers can use async fsync safely because leader durability guarantees safety /// /// # Failure Semantics /// - On error, implementer MAY roll back partial writes @@ -252,7 +252,7 @@ pub trait RaftLog: Send + Sync + 'static { /// /// # Usage Pattern /// ```rust,ignore - /// // Leader with MemFirst strategy + /// // Leader with async fsync /// raft_log.append_entries(new_entries).await?; /// raft_log.wait_durable(max_index).await?; // MUST wait before RPC response /// respond_to_client(Ok(())); @@ -261,7 +261,7 @@ pub trait RaftLog: Send + Sync + 'static { /// # Safety Invariants /// - MUST NOT return until flush() for this index completes successfully /// - If implementation doesn't support async durability, return Ok(()) immediately - /// - Critical for MemFirst strategy correctness + /// - Critical for async-fsync correctness async fn wait_durable( &self, index: u64, diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index 9bcfaa07..43e955e9 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -12,15 +12,13 @@ use bytes::Bytes; use d_engine_proto::common::{Entry, EntryPayload}; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; /// Test context for BufferedRaftLog tests pub struct BufferedRaftLogTestContext { pub raft_log: Arc>, pub storage: Arc, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub instance_id: String, } @@ -28,7 +26,6 @@ pub struct BufferedRaftLogTestContext { impl BufferedRaftLogTestContext { /// Create a new test context with specified strategy and flush policy pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -37,7 +34,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -52,7 +48,6 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy, flush_policy, instance_id: instance_id.to_string(), } @@ -92,7 +87,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -105,7 +99,6 @@ impl BufferedRaftLogTestContext { let ctx = Self { raft_log, storage, - strategy: PersistenceStrategy::MemFirst, flush_policy, instance_id: instance_id.to_string(), }; @@ -120,7 +113,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -135,7 +127,6 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), instance_id: self.instance_id.clone(), } diff --git a/d-engine-server/src/network/grpc/grpc_raft_service.rs b/d-engine-server/src/network/grpc/grpc_raft_service.rs index 15940d48..5b66126c 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service.rs @@ -6,7 +6,6 @@ use crate::Node; use crate::proto_convert; use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; -use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::RaftOneshot; use d_engine_core::TypeConfig; #[cfg(feature = "watch")] @@ -136,12 +135,24 @@ where Pin> + Send>>; /// Processes a persistent bidirectional AppendEntries stream from the cluster leader. + /// #446: responses are forwarded as soon as each one is ready, not in strict arrival + /// order — leader-side match_index/next_index updates are already designed to + /// tolerate out-of-order pipeline responses (`leader_state.rs`, "only advance, + /// never retreat"), so nothing downstream needs strict ordering. Strict FIFO would + /// let one response still waiting on this node's own durable_index (RPO=0) block + /// every later, already-ready response on the same connection — including + /// unrelated ones like heartbeats. /// - /// Decouples request ingestion from response emission: - /// - recv task: reads batches from the stream, dispatches each as a `InboundEvent::AppendEntries` - /// (non-blocking between batches) - /// - forwarder task: drains ordered response handles sequentially; ordering is guaranteed - /// by the Raft single-threaded event loop + /// Single task, bounded concurrency: reads a new request only while fewer than + /// `max_pending_append_responses` requests are still in flight, so a stalled fsync + /// bounds memory/task growth instead of growing without limit. + /// + /// Known tradeoff, not an oversight: this is a single task, so a slow/stuck + /// network write (`out_tx.send().await` blocking because the peer isn't reading) + /// also delays reading new requests AND processing the shutdown signal, until the + /// write unblocks or the connection dies. Accepted deliberately — if the peer + /// isn't reading responses, there's no useful work to do by reading more requests + /// either; this is legitimate backpressure, not a bug. async fn stream_append_entries( &self, request: tonic::Request>, @@ -157,30 +168,31 @@ where let mut in_stream = request.into_inner(); let event_tx = self.event_tx.clone(); - let ordered_channel_capacity = self.node_config.raft.ordered_channel_capacity; + let max_pending = self.node_config.raft.max_pending_append_responses; let mut shutdown = self.shutdown_signal.clone(); + let node_id = self.node_id; - // Output: ordered ACKs sent back to the leader over the bidi stream - let (out_tx, out_rx) = mpsc::channel::>(128); - - // Ordered queue: response oneshot receivers in FIFO arrival order - let (ordered_tx, mut ordered_rx) = mpsc::channel::< - MaybeCloneOneshotReceiver>, - >(ordered_channel_capacity); + // Output: ACKs sent back to the leader over the bidi stream, in completion order. + // Capacity matches max_pending — completed responses can never outnumber + // in-flight requests, so there's no separate number to reason about here. + let (out_tx, out_rx) = mpsc::channel::>(max_pending); - // Recv task: read batches, dispatch to Raft loop without waiting for each ACK. - // Selects on shutdown signal so the task exits immediately on node stop, rather - // than waiting for the next message from the leader. This unblocks serve_with_shutdown - // and allows Arc (and Arc) to be released promptly after stop(). + // Single task: read requests, dispatch to the Raft loop, and forward whichever + // response becomes ready first — bounded by `max_pending` in-flight responses. tokio::spawn(async move { use futures::StreamExt; + use futures::stream::FuturesUnordered; + + let mut pending = FuturesUnordered::new(); + let mut inbound_open = true; + loop { tokio::select! { biased; _ = shutdown.changed() => { break; } - result = in_stream.next() => { + result = in_stream.next(), if inbound_open && pending.len() < max_pending => { match result { Some(Ok(req)) => { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); @@ -188,32 +200,51 @@ where debug!("[stream_append_entries|recv] event_tx closed"); break; } - if ordered_tx.send(resp_rx).await.is_err() { - break; - } + pending.push(async move { + match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(Status::internal("Response channel closed")), + } + }); } Some(Err(e)) => { // Debug: expected when the peer goes away (crash/restart/shutdown), self-heals. debug!("[stream_append_entries|recv] stream error: {:?}", e); - break; + inbound_open = false; } - None => break, + None => inbound_open = false, + } + } + Some(result) = pending.next(), if !pending.is_empty() => { + // Observability only — behavior doesn't change, the send still + // runs to completion normally. If the peer isn't reading (network + // stall, dead connection with the TCP timeout not yet fired), this + // surfaces it instead of silently blocking with zero signal. + let mut send_fut = std::pin::pin!(out_tx.send(result)); + let mut stuck_logged = false; + let closed = loop { + tokio::select! { + res = &mut send_fut => break res.is_err(), + _ = tokio::time::sleep(Duration::from_secs(5)), if !stuck_logged => { + stuck_logged = true; + error!( + node_id, + "stream_append_entries forwarder stuck sending a \ + response for >5s — peer may not be reading \ + (network stall or dead connection)" + ); + metrics::counter!( + "server.grpc.stream_append_entries.forwarder_stuck" + ) + .increment(1); + } + } + }; + if closed { + break; } } - } - } - }); - - // Forwarder task: drain ordered queue sequentially (FIFO guaranteed by Raft loop) - tokio::spawn(async move { - while let Some(resp_rx) = ordered_rx.recv().await { - let result = match resp_rx.await { - Ok(Ok(resp)) => Ok(resp), - Ok(Err(status)) => Err(status), - Err(_) => Err(Status::internal("Response channel closed")), - }; - if out_tx.send(result).await.is_err() { - break; } } }); diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs index 97d5583f..32f8de17 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs @@ -3,12 +3,15 @@ use std::time::Duration; use crate::ApplyResult; use d_engine_core::AppendResponseWithUpdates; use d_engine_core::InternalEvent; +use d_engine_core::MaybeCloneOneshot; +use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::MockElectionCore; use d_engine_core::MockMembership; use d_engine_core::MockRaftLog; use d_engine_core::MockReplicationCore; use d_engine_core::MockTypeConfig; use d_engine_core::RaftNodeConfig; +use d_engine_core::RaftOneshot; use d_engine_core::convert::safe_kv_bytes; use d_engine_proto::client::ClientReadRequest; use d_engine_proto::client::ClientWriteRequest; @@ -605,3 +608,175 @@ async fn test_handle_client_scan_not_leader_carries_leader_hint_in_metadata() { Some("http://127.0.0.1:9082") ); } + +/// Historical record, not a regression guard: this is the strict-FIFO forwarder +/// pattern `stream_append_entries` used *before* #446 (one withheld response blocked +/// every later response on the same connection). It reconstructs the old primitives +/// rather than calling production code, because that code no longer exists — +/// `stream_append_entries` was rewritten to a bounded, order-tolerant forwarder (see +/// `test_stream_append_entries_does_not_block_ready_response_behind_pending_one` for +/// the real, current behavior). Kept only so a future reader can see what the old +/// failure mode looked like; do not treat this as coverage of current code. +#[tokio::test] +async fn test_ordered_forwarder_head_of_line_blocking() { + let (out_tx, mut out_rx) = mpsc::channel::>(128); + let (ordered_tx, mut ordered_rx) = mpsc::channel::< + MaybeCloneOneshotReceiver>, + >(128); + + // Mirrors grpc_raft_service.rs's forwarder loop. + tokio::spawn(async move { + while let Some(resp_rx) = ordered_rx.recv().await { + let result = match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(tonic::Status::internal("Response channel closed")), + }; + if out_tx.send(result).await.is_err() { + break; + } + } + }); + + // First item: never resolved — stands in for a response withheld pending durable_index. + let (_stuck_tx, stuck_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(stuck_rx).await.unwrap(); + + // Second item: already resolved — stands in for an unrelated, ready-to-send response + // (e.g. a heartbeat) that arrived right after. + let (ready_tx, ready_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(ready_rx).await.unwrap(); + ready_tx.send(Ok(AppendEntriesResponse::success(1, 1, None))).unwrap(); + + // The second, already-ready response must not be observable yet — it's stuck + // behind the first, unresolved one in strict FIFO order. + let blocked = time::timeout(Duration::from_millis(50), out_rx.recv()).await; + assert!( + blocked.is_err(), + "an already-ready response was blocked behind an earlier unresolved one — \ + confirms the forwarder is strict FIFO" + ); +} + +/// #446: `stream_append_entries` must not let a response still withheld (durable_index +/// hasn't caught up to what it claims) block a later, unrelated response that's already +/// answerable. Drives the real production method end-to-end — not a reconstruction — +/// via a synthetic 2-item input stream. +/// +/// request 1 claims index 10 while `durable_index()` is fixed at 5 — withheld, +/// queued in `pending_append_acks`, never released in this test. +/// request 2 claims index 3, which is `<= durable_index` — answerable immediately. +/// +/// If the forwarder is still strict FIFO, the first item out of the response stream +/// would have to be request 1's (never arrives) — this test would time out. If it's +/// the new bounded/order-tolerant forwarder, request 2's response (identifiable by its +/// distinct `last_match.term` marker) comes out first. +#[tokio::test] +async fn test_stream_append_entries_does_not_block_ready_response_behind_pending_one() { + tokio::time::pause(); + let settings = RaftNodeConfig::new().expect("Should succeed to init RaftNodeConfig."); + let mut settings = settings.validate().expect("Validate RaftNodeConfig successfully"); + settings.raft.general_raft_timeout_duration_in_ms = 200; + settings.raft.batching.max_batch_size = 1; + + let mut membership = MockMembership::::new(); + membership.expect_voters().returning(Vec::new); + membership.expect_members().returning(Vec::new); + membership.expect_replication_peers().returning(Vec::new); + membership.expect_get_peers_id_with_condition().returning(|_| vec![]); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_load_hard_state().returning(|| Ok(None)); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + raft_log.expect_last_log_id().returning(|| None); + // Fixed durable frontier: only request 2's claimed index (3) clears it. + raft_log.expect_durable_index().returning(|| 5); + + let call_count = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + let mut replication_handler = MockReplicationCore::::new(); + replication_handler + .expect_check_append_entries_request_is_legal() + .returning(|my_term, _, _| AppendEntriesResponse::success(1, my_term, None)); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + let is_first = call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0; + let (claimed_index, term_marker) = if is_first { (10, 111) } else { (3, 222) }; + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + term_marker, + Some(LogId { + term: term_marker, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let builder = MockBuilder::new(graceful_rx); + let node = builder + .with_raft_log(raft_log) + .with_membership(membership) + .with_replication_handler(replication_handler) + .with_node_config(settings) + .build_node(); + node.set_rpc_ready(true); + + let raft_lock = node.raft_core.clone(); + let _raft_handle = tokio::spawn(async move { + let mut raft = raft_lock.lock().await; + let _ = time::timeout(Duration::from_secs(5), raft.run()).await; + }); + + tokio::time::advance(Duration::from_millis(2)).await; + tokio::time::sleep(Duration::from_millis(2)).await; + + // request 1: prev_log_index=0. request 2: prev_log_index=99 — deliberately different + // from request 1's, so merge_append_entries (which only merges contiguous requests) + // can never combine them into a single handle_append_entries call. + let req1 = AppendEntriesRequest { + term: 1, + leader_id: 1, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let req2 = AppendEntriesRequest { + prev_log_index: 99, + ..req1.clone() + }; + let stream = crate::test_utils::create_test_snapshot_stream(vec![req1, req2]); + + let response = node + .stream_append_entries(Request::new(stream)) + .await + .expect("stream_append_entries must accept the request"); + use futures::StreamExt; + let mut out_stream = response.into_inner(); + + let first_out = time::timeout(Duration::from_secs(2), out_stream.next()) + .await + .expect( + "the response for request 2 (already durable) must arrive without waiting for \ + request 1 (withheld) — if this times out, the forwarder is still strict FIFO", + ) + .expect("stream must yield an item") + .expect("must be Ok, not a transport error"); + + let last_match = match first_out.result { + Some(d_engine_proto::server::replication::append_entries_response::Result::Success( + success, + )) => success.last_match.expect("success response must carry last_match"), + other => panic!("expected a success response, got {other:?}"), + }; + assert_eq!( + last_match.term, 222, + "the first response observed must be request 2's (marker term=222) — request 1 \ + (marker term=111) is still withheld and must not be observed yet, nor block this one" + ); +} diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 6c3f85b5..0556b0e2 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -7,7 +7,6 @@ use d_engine_core::LogStore; use d_engine_core::MockStateMachine; use d_engine_core::MockStorageEngine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftNodeConfig; use d_engine_core::StateMachine; use d_engine_core::StorageEngine; @@ -58,7 +57,6 @@ async fn test_set_raft_log_replaces_default() { BufferedRaftLog::>::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/src/test_utils/integration/mod.rs b/d-engine-server/src/test_utils/integration/mod.rs index 838f4c9c..00365525 100644 --- a/d-engine-server/src/test_utils/integration/mod.rs +++ b/d-engine-server/src/test_utils/integration/mod.rs @@ -49,7 +49,6 @@ use d_engine_core::FlushPolicy; use d_engine_core::LogSizePolicy; use d_engine_core::MockStateMachine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftLog; use d_engine_core::RaftNodeConfig; use d_engine_core::ReplicationHandler; @@ -181,7 +180,6 @@ pub fn setup_raft_components( let (buffered_raft_log, receiver) = BufferedRaftLog::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/common/mod.rs b/d-engine-server/tests/common/mod.rs index 2d10210a..7dfd1b17 100644 --- a/d-engine-server/tests/common/mod.rs +++ b/d-engine-server/tests/common/mod.rs @@ -13,7 +13,6 @@ use d_engine_core::config::BackoffPolicy; use d_engine_core::config::ElectionConfig; use d_engine_core::config::FlushPolicy; use d_engine_core::config::PersistenceConfig; -use d_engine_core::config::PersistenceStrategy; use d_engine_core::config::RaftConfig; use d_engine_core::config::RaftNodeConfig; use d_engine_core::config::SnapshotConfig; @@ -124,7 +123,6 @@ pub async fn create_node_config( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 100, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -174,7 +172,6 @@ pub async fn create_node_config_with_role( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 1, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -218,7 +215,6 @@ pub fn node_config(cluster_toml: &str) -> RaftNodeConfig { ..Default::default() }, persistence: PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs index de85eebd..ef0f53e2 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs @@ -8,9 +8,7 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tokio::time::sleep; @@ -21,7 +19,6 @@ use super::TestContext; async fn test_crash_recovery() { // Create and populate storage let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -65,7 +62,6 @@ async fn test_crash_recovery() { async fn test_crash_recovery_with_multiple_entries() { // Create and populate storage let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -128,7 +124,6 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -165,7 +160,6 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -210,7 +204,6 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -259,7 +252,6 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -299,21 +291,18 @@ async fn test_recovery_under_different_scenarios() { // drain cycle, so all 100 entries are always durable after explicit flush(). let scenarios = vec![ ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, 100usize, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10, }, 100, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -321,9 +310,9 @@ async fn test_recovery_under_different_scenarios() { ), ]; - for (strategy, flush_policy, expected_recovery) in scenarios { - let instance_id = format!("recovery_test_{strategy:?}_{flush_policy:?}"); - let original_ctx = TestContext::new(strategy.clone(), flush_policy.clone(), &instance_id); + for (flush_policy, expected_recovery) in scenarios { + let instance_id = format!("recovery_test_{flush_policy:?}"); + let original_ctx = TestContext::new(flush_policy.clone(), &instance_id); // Add test data for i in 1..=100 { @@ -351,7 +340,7 @@ async fn test_recovery_under_different_scenarios() { assert_eq!( recovered_ctx.raft_log.len(), expected_recovery, - "Recovery mismatch for strategy {strategy:?} policy {flush_policy:?}" + "Recovery mismatch for policy {flush_policy:?}" ); recovered_ctx.close().await; } @@ -363,7 +352,6 @@ async fn test_memfirst_crash_recovery_durability() { let recovered_path = { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10000, }, @@ -390,7 +378,6 @@ async fn test_memfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -424,7 +411,6 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -458,7 +444,6 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index ad11cddb..8687f2a4 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -15,15 +15,14 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, alias::ROF, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog, alias::ROF}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tempfile::tempdir; mod crash_recovery_test; mod performance_test; +mod quorum_crash_recovery_test; mod storage_integration_test; mod stress_test; @@ -32,7 +31,6 @@ pub struct TestContext { pub raft_log: Arc>>, pub storage: Arc, pub _temp_dir: Option, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub path: String, } @@ -40,7 +38,6 @@ pub struct TestContext { impl TestContext { /// Create new test context with FileStorageEngine pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -51,7 +48,6 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 10000, shutdown_timeout_ms: 5000, @@ -67,7 +63,6 @@ impl TestContext { path: path.to_str().unwrap().to_string(), raft_log, storage, - strategy, flush_policy, _temp_dir: Some(temp_dir), } @@ -92,7 +87,6 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), max_buffered_entries: 10000, shutdown_timeout_ms: 5000, @@ -106,7 +100,6 @@ impl TestContext { Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), _temp_dir: Some(temp_dir), path: self.path.clone(), diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 453e17e7..1d050f53 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -6,9 +6,7 @@ use super::TestContext; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use std::collections::HashMap; @@ -33,7 +31,6 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -104,7 +101,6 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -171,7 +167,6 @@ mod filter_out_conflicts_and_append_performance_tests { async fn test_last_entry_id_performance() { // Set up test context let test_context = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 360_000, }, @@ -252,7 +247,6 @@ async fn test_performance_benchmarks() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -342,7 +336,6 @@ async fn test_read_performance_under_concurrent_write_load() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs new file mode 100644 index 00000000..351e5a8d --- /dev/null +++ b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs @@ -0,0 +1,108 @@ +//! Quorum + real-disk crash recovery integration test (#446 gap 4). +//! +//! Composes two pieces that are each already covered in isolation elsewhere, but never +//! together: `calculate_majority_matched_index` (RPO=0 quorum arithmetic, unit-tested +//! against a gated mock in `buffered_raft_log_test/quorum_durability_test.rs`) and real +//! `FileStorageEngine` crash/reopen (unit-tested without any quorum math in +//! `crash_recovery_test.rs`). This file proves they actually compose: an index that the +//! quorum calculation says is safe to acknowledge to the client is still there after a +//! real crash + reopen from the same on-disk path. +//! +//! Followers are represented as reported match_index values, same as in +//! `quorum_durability_test.rs` — this file's job is the leader-side real-disk durability +//! boundary, not follower ACK withholding (covered by follower_state_test.rs / +//! learner_state_test.rs). +//! +//! Deliberately NOT attempted here, and now CONFIRMED impossible with this engine's +//! architecture (not just a flakiness risk — an actual dead end, verified by building and +//! deadlocking it): proving that an entry which never reached quorum-durable is genuinely +//! absent from a real crash + reopen. `BufferedRaftLog::append_entries` +//! (`d-engine-core/src/storage/buffered_raft_log.rs:467-500`) is documented and +//! implemented to block the caller until `persist_entries()` returns — "still blocks the +//! caller until truly persisted" — and `FileLogStore::persist_entries` +//! (`d-engine-server/src/storage/adaptors/file/file_storage_engine.rs:254`) already writes +//! the entry to the real OS-visible file as an unconditional part of its body, before it +//! can return. So by the time `append_entries().await` ever resolves at all, the entry is +//! already on the file — there is no window where it's "acknowledged as appended" yet +//! "recoverably absent." A gate on `persist_entries()` was built and tried here; it did +//! not create the intended window, it just deadlocked `append_entries()` forever (the +//! call this file's other test depends on to make progress at all). Reverted. +//! What IS real and already correctly tested (see below): the gap between +//! `last_entry_id` and `durable_index` — `persist_entries()` writes the bytes, but a +//! *separate* `flush()` call (`sync_all()`) is what advances `durable_index`, and that one +//! genuinely runs later/independently. What's NOT reachable by a same-process test is +//! observing that an un-`sync_all`'d write doesn't survive — on the same OS instance, +//! `write()` alone (which `persist_entries` already does) is enough for a freshly-opened +//! handle to see the bytes, real crash or not. Proving the un-fsynced case would need an +//! actual power-loss simulation (dropped page cache / real reboot) — which prior sessions +//! already found to be a poor fit for this class of bug (see mempalace notes on the +//! Jepsen/lazyfs work for #444). + +use super::TestContext; +use d_engine_core::FlushPolicy; +use d_engine_core::RaftLog; + +#[tokio::test] +async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { + let ctx = TestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_quorum_ack_survives_crash", + ); + + // First 5 entries, explicitly flushed: genuinely durable, deterministic. + ctx.append_entries(1, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 5); + + // 3-node cluster: both followers already report match_index=5 (post-Stage2 + // semantics — a follower only reports a match_index once its own durable_index + // reaches it). This is the index the leader would actually acknowledge to the + // client. + let commit = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!( + commit, + Some(5), + "index 5 is durable on the leader and acked by both followers" + ); + + // A follower report of 10 must not move commit past what the leader itself has + // fsynced — restates the Stage1 invariant as this test's own setup precondition + // rather than assuming it silently. + let would_be_wrong = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![10, 5]); + assert_eq!( + would_be_wrong, + Some(5), + "leader's own un-fsynced tail must not leak into the client-visible commit index" + ); + + // Second batch, also explicitly flushed, so the whole log is durable before the + // simulated crash — keeps this test's crash/recovery assertions exact, not bounded. + ctx.append_entries(6, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 10); + + let recovered = ctx.recover_from_crash(); + ctx.close().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // The index that was actually acknowledged to the client (5) — and everything else + // that was durably flushed (up to 10) — survives the real crash + reopen. + assert_eq!(recovered.raft_log.durable_index(), 10); + for i in 1..=10 { + assert!( + recovered.raft_log.entry(i).unwrap().is_some(), + "entry {i} must survive real crash + reopen" + ); + } + + // Re-running the same quorum calculation against the recovered log reaches the same + // conclusion — the leader's durability contribution to quorum is stable across a + // real restart, not just in the pre-crash in-memory view. + let commit_after_recovery = + recovered.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!(commit_after_recovery, Some(5)); + + recovered.close().await; +} diff --git a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs index 8cd4ab85..f9c9dfe6 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs @@ -3,7 +3,7 @@ //! These tests verify BufferedRaftLog integration with FileStorageEngine //! at the storage layer, including compaction and storage-specific operations. -use d_engine_core::{FlushPolicy, PersistenceStrategy, RaftLog}; +use d_engine_core::{FlushPolicy, RaftLog}; use d_engine_proto::common::LogId; use super::TestContext; @@ -14,7 +14,6 @@ use super::TestContext; #[tokio::test] async fn test_log_compaction() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index 44972c8a..3f41a4c0 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -6,7 +6,7 @@ use std::time::Duration; use bytes::Bytes; -use d_engine_core::{FlushPolicy, LogStore, PersistenceStrategy, RaftLog, StorageEngine}; +use d_engine_core::{FlushPolicy, LogStore, RaftLog, StorageEngine}; use d_engine_proto::common::{Entry, EntryPayload}; use futures::future::join_all; use tokio::time::Instant; @@ -23,7 +23,6 @@ use super::TestContext; #[tokio::test] async fn test_high_concurrency() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -63,7 +62,6 @@ async fn test_high_concurrency() { #[traced_test] async fn test_high_concurrency_mixed_operations() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -142,7 +140,6 @@ mod mem_first_tests { #[tokio::test] async fn test_basic_write_before_persist() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -158,7 +155,6 @@ mod mem_first_tests { #[tokio::test] async fn test_async_persistence() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -177,7 +173,6 @@ mod mem_first_tests { #[tokio::test] async fn test_power_loss_data_loss() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -195,7 +190,6 @@ mod mem_first_tests { #[tokio::test] async fn test_high_concurrency_memory_only() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -232,7 +226,6 @@ mod mem_first_tests { #[tokio::test] async fn test_term_index_correctness_under_load() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs index b5edc3e4..051bd020 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs @@ -125,8 +125,6 @@ learner_check_throttle_ms = 100 election_timeout_min = 300 election_timeout_max = 3000 -[raft.persistence] -strategy = "MemFirst" [retry.election] max_retries = 5 diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index 96cbe01a..b5432c04 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -59,7 +59,6 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" # Only strategy in v0.2.4+ (DiskFirst removed) flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } max_buffered_entries = 10000 ``` @@ -123,7 +122,7 @@ All performance reports in `/benches/standalone-bench/reports` use this exact co **Raft settings:** -- Persistence: `MemFirst` (only strategy in v0.2.4+) with 1000ms idle flush interval +- Persistence: batched fsync (Level 3, fdatasync) with 1000ms idle flush interval - Read consistency: `LeaseRead` (500ms lease duration) - Replication: Batched append entries (5000 threshold, 0ms delay) - Network: Tuned for high throughput (see `config/n1.toml` for details) diff --git a/d-engine/src/docs/performance/throughput-optimization-guide.md b/d-engine/src/docs/performance/throughput-optimization-guide.md index 03bc4efc..1fe3341e 100644 --- a/d-engine/src/docs/performance/throughput-optimization-guide.md +++ b/d-engine/src/docs/performance/throughput-optimization-guide.md @@ -23,18 +23,6 @@ pub(crate) enum ConnectionType { ## Persistence Strategy & Throughput/Latency Trade-offs -`MemFirst` is the only persistence strategy in v0.2.4+. It batches writes to OS page cache and flushes with fsync asynchronously — committing data to disk before notifying Raft. - -### Strategy Configuration - -```toml -[raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -``` - -### `MemFirst` Strategy - - **Write Path**: Entries are written to OS page cache via `db.write()` / `file.write()`; the IO thread batches them and calls fsync (`flush_wal(true)` / `sync_all()`) before advancing `durable_index`. Raft only counts an entry toward quorum after fsync completes. @@ -54,8 +42,7 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } - Lower values reduce the unflushed batch window but increase IO pressure. - Default `1000` ms is suitable for most workloads. -> **Note**: `DiskFirst` strategy was removed in v0.2.4. `MemFirst` replaced it with batched -> fsync — multiple writes share one fsync call, reducing IO overhead while still providing +> **Note**: Writes are batched into a single fsync, reducing IO overhead while still providing > disk-level durability for all client-acknowledged (committed) writes. ## Batching Configuration @@ -179,7 +166,7 @@ tonic::transport::Server::builder() ``` -Inbound message size is the one setting that *is* per-service rather than +Inbound message size is the one setting that _is_ per-service rather than transport-wide, so it's applied on each `XxxServiceServer` individually: ```rust,ignore @@ -197,7 +184,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) | p99.9 Latency | 14015 µs | 11279 µs | -19.5% | > **Key improvement**: 15% reduction in tail latency - critical for consensus stability -> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning the PersistenceStrategy for your specific workload. +> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning `FlushPolicy` for your specific workload. > > For absolute performance benchmarks, see [v0.2.4 Performance Report](https://github.com/deventlab/d-engine/tree/main/benches/reports/v0.2.4/bench_report_v0.2.4.md) @@ -230,7 +217,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) ``` -5. **Monitor Flush Lag**: When using `MemFirst`, monitor the difference between `last_log_index` and `durable_index`. A growing gap indicates the disk is not keeping up with writes, increasing potential data loss. +5. **Monitor Flush Lag**: Monitor the difference between `last_log_index` and `durable_index`. Raft only counts an entry toward quorum and acknowledges it to the client after fsync — so a growing gap does not put acknowledged writes at risk. It does mean client-facing write latency is growing, and (if the gap keeps growing) the amount of work an unflushed batch would need to redo on restart is growing too. ## Anti-Patterns to Avoid @@ -244,11 +231,9 @@ get_peer_channel(peer_id, ConnectionType::Control).await?; client.request_vote(...) // DON'T: Set idle_flush_interval_ms too low — defeats batching. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1 } } // Near-synchronous; low throughput // DO: Use a generous idle interval to amortize disk I/O cost. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } ``` @@ -261,8 +246,8 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } Control: Low latency ↔ Data: High throughput ↔ Bulk: Bandwidth 3. **Improves fault containment** Connection issues affect only one operation type -4. **Decouples Performance from Durability** - `MemFirst` with tunable `idle_flush_interval_ms` lets you balance write throughput against flush frequency. +4. **Decouples Performance from Ack Latency** + Client-acknowledged writes are always fsync-durable — that's not tunable. `idle_flush_interval_ms` lets you balance write throughput against how long a client waits for that fsync. ## Reference Deployment Configurations @@ -277,7 +262,6 @@ Adjust values based on snapshot size, log append rate, and cluster size. ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -306,7 +290,6 @@ max_concurrent_streams = 128 ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -325,7 +308,7 @@ max_concurrent_streams = 256 ``` -**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput with a reasonable data loss window. +**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput; acknowledged writes are never at risk regardless of the interval, only ack latency and unflushed-batch replay time on restart scale with it. ### 3. 5-Node High-Durability Cluster (Production) @@ -335,8 +318,7 @@ max_concurrent_streams = 256 ```toml [raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush for durability +flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush, shorter ack latency [network.control] connection_window_size = 4_194_304 # 4MB @@ -354,7 +336,7 @@ max_concurrent_streams = 512 ``` -**Tip**: For higher write persistence within a process lifecycle, lower `idle_flush_interval_ms` (e.g., 100ms). Note: `MemFirst` is not power-loss safe regardless of flush interval. +**Tip**: Lower `idle_flush_interval_ms` (e.g., 100ms) shortens client-facing write latency and shrinks the unflushed-batch window an IO thread has to redo on restart. Acknowledged writes are power-loss safe regardless of this setting — Raft only counts an entry toward quorum, and acknowledges it to the client, after fsync completes. ## Network Environment Tuning Recommendations @@ -362,12 +344,12 @@ These parameters are primarily **network-dependent**, not CPU/memory dependent. Adjust them based on latency, packet loss, and connection stability. -| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | -| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ------------------------------------------------------- | -| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | -| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | -| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | -| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | +| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | +| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ----------------------------------------------------- | +| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | +| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | +| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | +| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | **Guidelines:** diff --git a/d-engine/src/docs/server_guide/customize-storage-engine.md b/d-engine/src/docs/server_guide/customize-storage-engine.md index 0874acb0..ef9b088b 100644 --- a/d-engine/src/docs/server_guide/customize-storage-engine.md +++ b/d-engine/src/docs/server_guide/customize-storage-engine.md @@ -91,7 +91,7 @@ impl StorageEngine for CustomStorageEngine { - **Consistency**: Maintain exactly-once semantics for log entries - **Performance**: Target >100k ops/sec for log persistence. Do not call `fsync` inside `persist_entries()`—the framework batches entries and calls `flush()` once per batch - (`MemFirst + FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. + (`FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. - **Resource Management**: Clean up resources in `Drop` implementation ## 3. StorageEngine API Reference diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index 712a3935..d036239d 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -15,7 +15,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -43,11 +43,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/single-node-expansion/config/n2.toml b/examples/single-node-expansion/config/n2.toml index 5e5356c3..bad8ce26 100644 --- a/examples/single-node-expansion/config/n2.toml +++ b/examples/single-node-expansion/config/n2.toml @@ -28,7 +28,6 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } max_buffered_entries = 10000 diff --git a/examples/single-node-expansion/config/n3.toml b/examples/single-node-expansion/config/n3.toml index 75a585e3..67cf1fd2 100644 --- a/examples/single-node-expansion/config/n3.toml +++ b/examples/single-node-expansion/config/n3.toml @@ -30,7 +30,6 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } max_buffered_entries = 10000 diff --git a/examples/sled-cluster/config/n1.toml b/examples/sled-cluster/config/n1.toml index a5b391e8..5902a6ad 100644 --- a/examples/sled-cluster/config/n1.toml +++ b/examples/sled-cluster/config/n1.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n2.toml b/examples/sled-cluster/config/n2.toml index 87a5f29a..c7a89b70 100644 --- a/examples/sled-cluster/config/n2.toml +++ b/examples/sled-cluster/config/n2.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n3.toml b/examples/sled-cluster/config/n3.toml index 5fe5b628..c099227a 100644 --- a/examples/sled-cluster/config/n3.toml +++ b/examples/sled-cluster/config/n3.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } diff --git a/examples/three-nodes-embedded/README.md b/examples/three-nodes-embedded/README.md index 25255a59..b6f2dda2 100644 --- a/examples/three-nodes-embedded/README.md +++ b/examples/three-nodes-embedded/README.md @@ -84,7 +84,6 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { threshold = 100, interval_ms = 20 } } ``` diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index a040e1ae..bb6db45a 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 20e816f2..95280d76 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 0a29fe2d..4aa67b3b 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n1.toml b/examples/three-nodes-standalone/docker/config/n1.toml index efa8e941..8deff325 100644 --- a/examples/three-nodes-standalone/docker/config/n1.toml +++ b/examples/three-nodes-standalone/docker/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n2.toml b/examples/three-nodes-standalone/docker/config/n2.toml index 78f66899..7e7eb644 100644 --- a/examples/three-nodes-standalone/docker/config/n2.toml +++ b/examples/three-nodes-standalone/docker/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n3.toml b/examples/three-nodes-standalone/docker/config/n3.toml index 674109a2..73d6bc59 100644 --- a/examples/three-nodes-standalone/docker/config/n3.toml +++ b/examples/three-nodes-standalone/docker/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] From d9a5b1730cbfc7d244cf6043cde7116bfe8dc231 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:12:56 +0800 Subject: [PATCH 3/4] chore #446: bump grpc-go and x/net to resolve Dependabot alerts (#86-91) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upgrade google.golang.org/grpc v1.80.0→v1.83.2 (d-engine-proto/go, examples/quick-start-standalone) - Upgrade golang.org/x/net v0.53.0→v0.58.0 in both modules - Fixes: xDS RBAC authz bypass, HTTP/2 rapid-reset DoS, RBAC parser panic, HTTP/2 DATA frame OOM, x/net HTML parser DoS --- d-engine-proto/go/go.mod | 10 +++---- d-engine-proto/go/go.sum | 40 +++++++++++++------------- examples/quick-start-standalone/go.mod | 10 +++---- examples/quick-start-standalone/go.sum | 40 +++++++++++++------------- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/d-engine-proto/go/go.mod b/d-engine-proto/go/go.mod index 68f93120..b5dd4aa2 100644 --- a/d-engine-proto/go/go.mod +++ b/d-engine-proto/go/go.mod @@ -3,13 +3,13 @@ module github.com/deventlab/d-engine/proto go 1.25.0 require ( - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect ) diff --git a/d-engine-proto/go/go.sum b/d-engine-proto/go/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/d-engine-proto/go/go.sum +++ b/d-engine-proto/go/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/examples/quick-start-standalone/go.mod b/examples/quick-start-standalone/go.mod index 79b1ef9c..9865cf61 100644 --- a/examples/quick-start-standalone/go.mod +++ b/examples/quick-start-standalone/go.mod @@ -6,13 +6,13 @@ replace github.com/deventlab/d-engine/proto => ../../d-engine-proto/go require ( github.com/deventlab/d-engine/proto v0.0.0 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/examples/quick-start-standalone/go.sum b/examples/quick-start-standalone/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/examples/quick-start-standalone/go.sum +++ b/examples/quick-start-standalone/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= From 97eb1c0337006c4e838ecfbecef89b1d42978477 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:46:01 +0800 Subject: [PATCH 4/4] fix #446: single-writer durable_index/persisted_index, close truncation TOCTOU race - durable_index: sole writer is raft.rs's event loop, content-validated via try_advance_durable_index(index, term) against entry_term(index) - persisted_index: sole writer is the IO thread, clamp moved from remove_range into IOTask::ReplaceRange - remove_range keeps its synchronous durable_index clamp (flush() short-circuit depends on it) - rename handle_non_write_cmd -> run_storage_tasks, max_index -> memory_max_index - remove dead config max_buffered_entries + 12 example/bench TOML configs - test: 22 d-engine-core + 5 d-engine-server tests updated for the new drain-fsync-completions pattern; new content_validated_watermark_test.rs --- benches/embedded-bench/config/n1.toml | 2 - benches/embedded-bench/config/n2.toml | 2 - benches/embedded-bench/config/n3.toml | 2 - d-engine-core/src/config/raft.rs | 13 -- d-engine-core/src/event.rs | 8 + d-engine-core/src/raft.rs | 9 + .../src/storage/buffered_raft_log.rs | 142 ++++++------ .../concurrent_fsync_test.rs | 99 ++++----- .../content_validated_watermark_test.rs | 204 ++++++++++++++++++ .../drain_fsync_test.rs | 33 +-- .../durable_index_test.rs | 5 +- .../flush_strategy_test.rs | 15 +- .../id_allocation_test.rs | 1 - .../performance_test.rs | 3 - .../persisted_index_clamp_test.rs | 4 +- .../pipeline_overlap_test.rs | 1 - .../process_crash_safety_test.rs | 1 - .../quorum_durability_test.rs | 12 +- .../raft_properties_test.rs | 12 +- .../replace_range_fsync_test.rs | 6 +- .../buffered_raft_log_test/shutdown_test.rs | 3 - .../truncation_fsync_fence_test.rs | 1 - .../src/storage/fsync_coordinator.rs | 27 +-- .../src/storage/fsync_coordinator_test.rs | 76 ++++++- d-engine-core/src/storage/raft_log.rs | 12 ++ .../buffered_raft_log_test_helpers.rs | 48 ++++- .../test_utils/mock/mock_storage_engine.rs | 4 +- d-engine-server/src/node/builder_test.rs | 1 - .../src/test_utils/integration/mod.rs | 1 - .../crash_recovery_test.rs | 10 +- .../tests/storage_buffered_raft_log/mod.rs | 27 ++- .../performance_test.rs | 2 - .../quorum_crash_recovery_test.rs | 4 +- .../storage_integration_test.rs | 3 +- .../storage_buffered_raft_log/stress_test.rs | 6 +- .../docs/examples/three-nodes-standalone.md | 1 - examples/single-node-expansion/config/n1.toml | 2 - examples/single-node-expansion/config/n2.toml | 1 - examples/single-node-expansion/config/n3.toml | 1 - .../three-nodes-standalone/config/n1.toml | 2 - .../three-nodes-standalone/config/n2.toml | 2 - .../three-nodes-standalone/config/n3.toml | 2 - .../docker/config/n1.toml | 2 - .../docker/config/n2.toml | 2 - .../docker/config/n3.toml | 2 - 45 files changed, 561 insertions(+), 255 deletions(-) create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs diff --git a/benches/embedded-bench/config/n1.toml b/benches/embedded-bench/config/n1.toml index 8dec8c84..da9af610 100644 --- a/benches/embedded-bench/config/n1.toml +++ b/benches/embedded-bench/config/n1.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n2.toml b/benches/embedded-bench/config/n2.toml index 455effff..1e10d048 100644 --- a/benches/embedded-bench/config/n2.toml +++ b/benches/embedded-bench/config/n2.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n3.toml b/benches/embedded-bench/config/n3.toml index 1296b03d..96f32af8 100644 --- a/benches/embedded-bench/config/n3.toml +++ b/benches/embedded-bench/config/n3.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 8e03f2f5..60a7456a 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -847,13 +847,6 @@ pub struct PersistenceConfig { #[serde(default = "default_flush_policy")] pub flush_policy: FlushPolicy, - /// Maximum number of in-memory log entries to buffer when using async strategies - /// - /// This acts as a safety valve to prevent memory exhaustion during periods of - /// high write throughput or when disk persistence is slow. - #[serde(default = "default_max_buffered_entries")] - pub max_buffered_entries: usize, - /// Maximum time to wait, on shutdown, for an in-flight fsync task to finish /// before giving up. Bounds close() against a stuck/slow disk — the task /// itself is not cancelled, it keeps running in the background regardless. @@ -871,11 +864,6 @@ fn default_flush_policy() -> FlushPolicy { } } -/// Default maximum buffered log entries -fn default_max_buffered_entries() -> usize { - 10_000 -} - fn default_shutdown_timeout_ms() -> u64 { 5_000 } @@ -904,7 +892,6 @@ impl Default for PersistenceConfig { fn default() -> Self { Self { flush_policy: default_flush_policy(), - max_buffered_entries: default_max_buffered_entries(), shutdown_timeout_ms: default_shutdown_timeout_ms(), } } diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 3f35dbf3..9c756914 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -71,6 +71,14 @@ pub enum InternalEvent { durable_index: u64, }, + /// Raw fsync-completion signal — NOT yet validated. Consumer must call + /// `raft_log().try_advance_durable_index(index, term)`, which re-checks + /// content before actually advancing `durable_index`. + FsyncCompleted { + index: u64, + term: u64, + }, + /// AppendEntries result from a per-follower ReplicationWorker back to the Raft loop. /// Leader processes this in handle_append_result: updates match_index, re-calculates commit, /// and drains pending_client_writes when quorum is achieved. diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index c8e2eac6..853c81eb 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -609,6 +609,15 @@ where .handle_log_flushed(durable_index, &self.ctx, &self.internal_event_tx) .await; } + InternalEvent::FsyncCompleted { index, term } => { + if let Some(new_durable) = + self.ctx.raft_log().try_advance_durable_index(index, term) + { + self.role + .handle_log_flushed(new_durable, &self.ctx, &self.internal_event_tx) + .await; + } + } InternalEvent::AppendResult { follower_id, result, diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 0fb7960d..2a0519aa 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -18,7 +18,7 @@ //! ## IO thread (notify-then-spawn-fsync) //! //! On wakeup from `write_notify`: -//! 1. **Read** — scan SkipMap range `(durable_index, max_index]` +//! 1. **Read** — scan SkipMap range `(durable_index, memory_max_index]` //! 2. **Persist** — write range to OS page cache via `persist_entries` //! 3. **Spawn fsync** — dispatch fdatasync to `spawn_blocking` pool via `spawn_fsync`, return immediately //! 4. **Loop** — back to `select!` for next wakeup; prior fsync runs concurrently in pool @@ -277,8 +277,8 @@ where // --- In-memory index --- // O(1) answer to "is this index currently held in memory" — lets callers // (e.g. entry_term()) reject an out-of-range index without touching `entries`. - min_index: AtomicU64, // Smallest log index (0 if empty) - max_index: AtomicU64, // Largest log index (0 if empty) + min_index: AtomicU64, // Smallest log index (0 if empty) + memory_max_index: AtomicU64, // Largest log index held in memory (0 if empty) — may be ahead of what's persisted/durable // The term of the last entry ever purged (compacted away after a snapshot). // Raft's AppendEntries consistency check needs the term at prev_log_index @@ -286,7 +286,7 @@ where // this, a follower can't tell "purged, but we agree" apart from "conflict". // // Must be published in the same critical section as the entries removal - // and the min_index/max_index advance it corresponds to — a reader must + // and the min_index/memory_max_index advance it corresponds to — a reader must // never be able to observe the entry gone but this boundary not yet set. last_purged_index: AtomicU64, last_purged_term: AtomicU64, @@ -342,7 +342,7 @@ where } fn last_entry_id(&self) -> u64 { - self.max_index.load(Ordering::Acquire) + self.memory_max_index.load(Ordering::Acquire) } fn durable_index(&self) -> u64 { @@ -394,7 +394,7 @@ where entry_id: u64, ) -> Option { // Bounds check: skip TermSegments entirely for out-of-range queries. - let max = self.max_index.load(Ordering::Acquire); + let max = self.memory_max_index.load(Ordering::Acquire); let min = self.min_index.load(Ordering::Acquire); if max == 0 || entry_id < min || entry_id > max { // Cold path: check purge boundary so that AppendEntries built with @@ -599,7 +599,7 @@ where if diverge_index <= last_current_index { // Real term conflict: truncate from diverge_index, replace with tail. // Await the done channel so callers can flush() knowing the truncation - // is durable — durable_index may exceed max_index after truncation, + // is durable — durable_index may exceed memory_max_index after truncation, // which would cause flush() to short-circuit before the replace lands. self.remove_range(diverge_index..=u64::MAX); self.insert_to_memory(tail); @@ -682,9 +682,16 @@ where self.purge_prefix(cutoff_index); // Purged entries are backed by the snapshot; treat cutoff as durable. - // Must run after purge_prefix() — advance_durable_and_notify() validates - // against last_purged_index, which purge_prefix() just established. - self.advance_durable_and_notify(cutoff_index.index); + // Already running on the single owner (called from role_state.rs, same + // thread as remove_range) — safe to apply directly, no message hop needed. + if let Some(new_durable) = + self.try_advance_durable_index(cutoff_index.index, cutoff_index.term) + && let Some(ref tx) = self.log_flush_tx + { + let _ = tx.send(crate::InternalEvent::LogFlushed { + durable_index: new_durable, + }); + } // Route purge through the IO thread so it never blocks the inbound event loop. // Also writes the purge boundary to META_CF in the RocksDB implementation. @@ -702,12 +709,36 @@ where Ok(()) } + fn try_advance_durable_index( + &self, + index: u64, + term: u64, + ) -> Option { + let prev = self.durable_index.load(Ordering::Acquire); + if index <= prev { + return None; + } + if self.entry_term(index) != Some(term) { + return None; + } + let safe = index.min( + self.memory_max_index + .load(Ordering::Acquire) + .max(self.last_purged_index.load(Ordering::Acquire)), + ); + if safe <= prev { + return None; + } + self.durable_index.fetch_max(safe, Ordering::AcqRel); + Some(safe) + } + async fn flush(&self) -> Result<()> { - let max_index = self.max_index.load(Ordering::Acquire); - if max_index == 0 { + let memory_max_index = self.memory_max_index.load(Ordering::Acquire); + if memory_max_index == 0 { return Ok(()); } - if self.durable_index.load(Ordering::Acquire) >= max_index { + if self.durable_index.load(Ordering::Acquire) >= memory_max_index { return Ok(()); } let (tx, rx) = oneshot::channel(); @@ -830,7 +861,7 @@ where // Initialize atomic boundaries let min_index = entries.front().map(|e| *e.key()).unwrap_or(0); - let max_index = entries.back().map(|e| *e.key()).unwrap_or(0); + let memory_max_index = entries.back().map(|e| *e.key()).unwrap_or(0); if disk_len > 0 && loaded_count == 0 { warn!( @@ -859,7 +890,7 @@ where shutdown_timeout_ms, entries: RwLock::new(entries), min_index: AtomicU64::new(min_index), - max_index: AtomicU64::new(max_index), + memory_max_index: AtomicU64::new(memory_max_index), last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), @@ -937,7 +968,7 @@ where /// from one-per-write to one-per-burst. /// /// On each wakeup: - /// 1. Read entries in `(durable_index, max_index]` from SkipMap. + /// 1. Read entries in `(durable_index, memory_max_index]` from SkipMap. /// 2. persist_entries to OS page cache (no fsync). /// 3. Drain any pending control commands from the mpsc channel. /// 4. fsync once — advance durable_index, wake WaitDurable callers. @@ -972,7 +1003,7 @@ where IOTask::Shutdown => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), true).await, IOTask::Flush(reply) => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, vec![reply], false).await, cmd => { - if Self::handle_non_write_cmd(cmd, &this, &mut pending_max).await { + if Self::run_storage_tasks(cmd, &this, &mut pending_max).await { break; } continue; @@ -1023,7 +1054,7 @@ where } IOTask::Flush(reply) => replies.push(reply), cmd => { - if Self::handle_non_write_cmd(cmd, this, pending_max).await { + if Self::run_storage_tasks(cmd, this, pending_max).await { for reply in replies { let _ = reply .send(Err(Error::Fatal("fatal IO error, batch aborted".into()))); @@ -1042,25 +1073,22 @@ where seen_shutdown } - /// Handle IOTask variants that are NOT `Flush` or `Shutdown`. - /// - /// Callers (`batch_processor`) dispatch `Flush` and `Shutdown` directly in the outer - /// `match` before this function is ever called — those two arms are unreachable here. + /// Runs one storage-mutating IOTask (Persist/ReplaceRange/Purge/Reset) + /// against log_store. Flush/Shutdown are intercepted by the caller + /// (`batch_processor`) before this is called — unreachable here. /// /// Returns `true` if `batch_processor` must exit immediately (fatal IO error). - async fn handle_non_write_cmd( + async fn run_storage_tasks( cmd: IOTask, this: &Arc, pending_max: &mut u64, ) -> bool { match cmd { IOTask::Flush(_) => { - unreachable!( - "Flush must be intercepted in the drain loop before handle_non_write_cmd" - ) + unreachable!("Flush must be intercepted in the drain loop before run_storage_tasks") } IOTask::Shutdown => { - unreachable!("Shutdown is always filtered out before reaching handle_non_write_cmd") + unreachable!("Shutdown is always filtered out before reaching run_storage_tasks") } IOTask::Persist { entries, done } => { if this.is_poisoned() { @@ -1077,7 +1105,7 @@ where } if max_idx > 0 { let current_bound = this - .max_index + .memory_max_index .load(Ordering::Acquire) .max(this.last_purged_index.load(Ordering::Acquire)); let safe_max_idx = max_idx.min(current_bound); @@ -1110,8 +1138,14 @@ where let _ = done.send(result); return true; // signal batch_processor to exit — disk state is corrupted } + // persisted_index moved here from remove_range — this handler + // is the sole writer now, single-threaded, no content check needed. + this.persisted_index + .fetch_min(truncate_from.saturating_sub(1), Ordering::AcqRel); + if max_idx > 0 { *pending_max = (*pending_max).max(max_idx); + this.persisted_index.fetch_max(max_idx, Ordering::AcqRel); this.fsync_coordinator.submit(this, max_idx, vec![]); } let _ = done.send(result); @@ -1165,7 +1199,7 @@ where // Reset boundaries self.min_index.store(0, Ordering::Release); - self.max_index.store(0, Ordering::Release); + self.memory_max_index.store(0, Ordering::Release); // Clear term indexes to ensure consistency after reset self.term_first_index.clear(); @@ -1220,9 +1254,9 @@ where } if let Some(last_entry) = entries.last() { - let mut current_max = self.max_index.load(Ordering::Relaxed); + let mut current_max = self.memory_max_index.load(Ordering::Relaxed); while last_entry.index > current_max { - match self.max_index.compare_exchange_weak( + match self.memory_max_index.compare_exchange_weak( current_max, last_entry.index, Ordering::AcqRel, @@ -1235,32 +1269,15 @@ where } } - // The single choke point every reported max_index must pass through — - // re-validates against the current log boundary regardless of how many - // upstream call sites raced to produce this value. - pub(super) fn advance_durable_and_notify( + /// Fire-and-forget signal to the single owner (raft.rs's event loop). + /// Called by fsync_coordinator (C) — never writes `durable_index` itself. + pub(super) fn notify_fsync_completed( &self, - reported_max: u64, + index: u64, + term: u64, ) { - let current_max = self - .max_index - .load(Ordering::Acquire) - .max(self.last_purged_index.load(Ordering::Acquire)); - let safe_max = reported_max.min(current_max); - debug_assert!( - safe_max == reported_max, - "advance_durable_and_notify: reported_max {reported_max} exceeded current bound {current_max}, clamped" - ); - if safe_max == 0 { - return; - } - let prev = self.durable_index.fetch_max(safe_max, Ordering::AcqRel); - if safe_max > prev - && let Some(ref tx) = self.log_flush_tx - { - let _ = tx.send(crate::InternalEvent::LogFlushed { - durable_index: safe_max, - }); + if let Some(ref tx) = self.log_flush_tx { + let _ = tx.send(crate::InternalEvent::FsyncCompleted { index, term }); } } @@ -1309,9 +1326,8 @@ where let entries = self.entries.write(); let (new_min, new_max) = self.remove_range_locked(&entries, range); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); - self.persisted_index.fetch_min(new_max, Ordering::AcqRel); self.durable_index.fetch_min(new_max, Ordering::AcqRel); // Clamps pending_max and bumps generation, in that order — see // fence_truncation()'s doc comment for why the order matters. @@ -1398,7 +1414,7 @@ where /// Purge entries at/below `cutoff.index`, publishing `last_purged_index`/ /// `last_purged_term` in the SAME critical section as the entries removal - /// and the min_index/max_index advance. Only place that should ever write + /// and the min_index/memory_max_index advance. Only place that should ever write /// `last_purged_*` — a reader must never observe the entries gone but the /// boundary not yet recorded (#442). pub fn purge_prefix( @@ -1409,7 +1425,7 @@ where let (new_min, new_max) = self.remove_range_locked(&entries, 0..=cutoff.index); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); // Write term before index (Release) so readers that load index first // then term (Acquire) always observe a consistent pair. @@ -1471,11 +1487,11 @@ where } #[cfg(test)] - pub(super) fn set_max_index_for_test( + pub(super) fn set_memory_max_index_for_test( &self, value: u64, ) { - self.max_index.store(value, Ordering::Release); + self.memory_max_index.store(value, Ordering::Release); } } @@ -1585,6 +1601,10 @@ mod term_segments_test; #[path = "buffered_raft_log_test/truncation_fsync_fence_test.rs"] mod truncation_fsync_fence_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] +mod content_validated_watermark_test; + #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 8cde96b9..495aad69 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -6,9 +6,9 @@ //! advance_durable_and_notify contract //! - **Concurrency**: Reset races, out-of-order completion, crash recovery +use crate::test_utils::drain_and_apply_fsync_completions; use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockStorageEngine, MockTypeConfig, - PersistenceConfig, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; use std::sync::Arc; @@ -37,12 +37,15 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index — see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -70,12 +73,13 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { "durable_index must not advance before fsync completes" ); - // Release the gate — flush() returns, advance_durable_and_notify(1) fires. + // Release the gate — flush() returns, notify_fsync_completed(1, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), @@ -117,12 +121,15 @@ async fn test_majority_matched_index_uses_durable_not_memory() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index — see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -175,12 +182,13 @@ async fn test_majority_matched_index_uses_durable_not_memory() { when a follower already reports it" ); - // Release the gate — flush() returns, advance_durable_and_notify(2) fires. + // Release the gate — flush() returns, notify_fsync_completed(2, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), @@ -223,7 +231,6 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -282,24 +289,24 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { // ── Logic correctness ───────────────────────────────────────────────────────── -/// `advance_durable_and_notify` is monotonic: a late-arriving lower index is a no-op. +/// `try_advance_durable_index` is monotonic: a late-arriving lower index is a no-op. /// -/// Directly call `advance_durable_and_notify(150)`, then `advance_durable_and_notify(100)`. +/// Directly call `try_advance_durable_index(150, 1)`, then `try_advance_durable_index(100, 1)`. /// Assert: /// - final `durable_index() == 150` (not 100) -/// - `LogFlushed` event fired exactly once (for 150), not twice +/// - the 150 call returns `Some(150)` (it fired), the 100 call returns `None` (no-op) /// /// Verifies the `fetch_max` invariant that makes out-of-order concurrent fsyncs safe. /// -/// Expected: -/// - After `advance_durable_and_notify(150)`: `durable_index() == 150`. -/// - After the subsequent `advance_durable_and_notify(100)`: `durable_index()` -/// is STILL `150` (unchanged — 100 < 150 must be a no-op, not a regression). -/// - The flush-completion notification fires exactly once, carrying 150 — the -/// discarded 100 call must not fire a second notification. +/// Test changed from #446/#447's original (which checked an `InternalEvent::LogFlushed` +/// on a channel): `try_advance_durable_index` no longer sends that notification itself — +/// the caller (raft.rs's `FsyncCompleted` handler) decides whether to fire +/// `handle_log_flushed`, based on this method's `Option` return value. So "fired +/// exactly once, only for 150" is now asserted directly on the return values instead of +/// on a channel — same intent, moved to match where the behavior actually lives now. #[tokio::test] async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { - // Storage engine choice doesn't matter here — advance_durable_and_notify is + // Storage engine choice doesn't matter here — try_advance_durable_index is // called directly, bypassing the real fsync pipeline entirely. let storage = Arc::new(MockStorageEngine::with_id( "durable_index_monotonic_when_fsyncs_complete_out_of_order".into(), @@ -310,44 +317,44 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, ); - let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel::(); - let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // advance_durable_and_notify() clamps against max_index — simulate a log - // that already has 150 entries, matching the highest value used below. - raft_log.set_max_index_for_test(150); + // try_advance_durable_index() content-validates against entry_term(index) — + // needs real entries in memory, not just a raw max_index poke. + let entries: Vec = (1..=150) + .map(|index| Entry { + index, + term: 1, + payload: None, + }) + .collect(); + raft_log.append_entries(entries).await.unwrap(); // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. - raft_log.advance_durable_and_notify(150); - raft_log.advance_durable_and_notify(100); + let result_150 = raft_log.try_advance_durable_index(150, 1); + let result_100 = raft_log.try_advance_durable_index(100, 1); + assert_eq!( + result_150, + Some(150), + "the 150 call must fire — it's the first advance" + ); + assert_eq!( + result_100, None, + "the later, lower 100 call must be a no-op (None), not a regression" + ); assert_eq!( raft_log.durable_index(), 150, "durable_index must reflect the highest index seen (150), not the \ later-arriving lower one (100)" ); - - // Exactly one LogFlushed event must have fired, carrying 150 — the - // no-op 100 call must not have sent a second event. - let event = log_flush_rx.try_recv().expect("LogFlushed must fire for the 150 call"); - match event { - InternalEvent::LogFlushed { durable_index } => { - assert_eq!(durable_index, 150, "LogFlushed must carry 150, not 100"); - } - other => panic!("expected InternalEvent::LogFlushed, got {other:?}"), - } - assert!( - log_flush_rx.try_recv().is_err(), - "no second LogFlushed event should have fired for the no-op 100 call" - ); } /// A `flush()` caller receives `Ok(())` only after its batch is physically on disk. @@ -377,7 +384,6 @@ async fn test_flush_caller_blocked_until_fsync_completes() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -450,7 +456,6 @@ async fn test_flush_callers_arriving_during_inflight_fsync_are_coalesced() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -533,7 +538,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_ok_reply() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -606,7 +610,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_err_reply() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -686,7 +689,6 @@ async fn test_reset_during_inflight_fsync_does_not_resurrect_stale_durable_index flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -767,12 +769,12 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready // Append triggers the automatic round (round 1), which wins the CAS and @@ -823,6 +825,7 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { result.is_ok(), "Y's flush() must succeed — its data was written after reset, not stale" ); + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), 1, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs new file mode 100644 index 00000000..68a38051 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs @@ -0,0 +1,204 @@ +//! Content-validated `durable_index` advance (#446/#447 single-owner +//! redesign). Tests `try_advance_durable_index(&self, index: u64, term: u64) +//! -> Option` — `Some(new_value)` only when it actually advanced, +//! `None` when rejected as stale (`entry_term(index) != Some(term)`) or +//! already applied. +//! +//! Not wired into the mod tree yet — add +//! `#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] +//! mod content_validated_watermark_test;` to `buffered_raft_log.rs` next to +//! the other test module declarations. +//! +//! Why these tests don't need thread races or timing gates (unlike +//! `truncation_fsync_fence_test.rs`): under single ownership, the report and +//! the truncation are just two sequential calls in whatever order they +//! happen to arrive — no interleaving *inside* a function body is possible +//! because there's only one caller. Each test below drives one arrival order +//! directly. +//! +//! `persisted_index` has no equivalent test here — it doesn't need content +//! validation. Its only writer is now the IO thread (B), processing +//! `IOTask::Persist`/`ReplaceRange` strictly in the order the single owner +//! (A) issued them (each `await`ed before the next is sent), so there's no +//! stale-message window the way there is for `durable_index`'s async, +//! un-awaited fsync-completion report. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +async fn new_raft_log() -> Arc> { + let storage = Arc::new(MockStorageEngine::with_id( + "content_validated_watermark_test".into(), + )); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + raft_log.start(receiver, None) +} + +/// Business scenario: follower has entries 1..=100 under term 1. A physical +/// fsync for "up to 100" is still in flight when a new leader (term 2) +/// truncates 81..=100 and replaces it with its own entries. The in-flight +/// fsync's completion — a report for (index=100, term=1) — arrives after the +/// replacement. Index 100 still exists, but it's term 2 now: the report +/// describes content that's gone. +/// +/// Expected: rejected. `durable_index` must not move to 100. +#[tokio::test] +async fn test_stale_durable_report_rejected_when_term_no_longer_matches() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // New leader (term 2) truncates 81..=100 and replaces with its own tail. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + // The stale in-flight fsync's report, generated before the truncation. + let result = raft_log.try_advance_durable_index(100, 1); + + assert_eq!( + result, None, + "a durable report for term=1 must be rejected once index 100 belongs to term=2" + ); + assert!( + raft_log.durable_index() < 81, + "durable_index ({}) must not advance into the replaced [81,100] range \ + on a rejected report", + raft_log.durable_index() + ); +} + +/// Sanity check: an unremarkable report (no truncation involved) must still +/// be applied. The new validation must not reject everything. +#[tokio::test] +async fn test_durable_report_accepted_when_term_still_matches() { + let raft_log = new_raft_log().await; + + let entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + let result = raft_log.try_advance_durable_index(100, 1); + + assert_eq!( + result, + Some(100), + "a report matching current log content must be applied" + ); + assert_eq!(raft_log.durable_index(), 100); +} + +/// Same scenario as `test_stale_durable_report_rejected_when_term_no_longer_matches`, +/// but the report arrives BEFORE the truncation instead of after — the other +/// possible arrival order. Under single ownership both orders must land on +/// the same final state, because the owner processes one event at a time +/// rather than racing a background write against a live update. +#[tokio::test] +async fn test_durable_report_then_truncation_is_order_independent() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // Report arrives first, while the log is still all term 1 — legitimately + // applied at this point in time. + let result = raft_log.try_advance_durable_index(100, 1); + assert_eq!(result, Some(100)); + assert_eq!(raft_log.durable_index(), 100); + + // Truncation arrives after — must still clamp durable_index down, + // exactly as it does today via `remove_range`'s existing fetch_min. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + assert!( + raft_log.durable_index() < 81, + "truncation must clamp durable_index down to 80 regardless of the \ + earlier report having advanced it to 100, durable_index is {}", + raft_log.durable_index() + ); +} + +/// Regression test for the `flush()` short-circuit (`durable_index >= +/// memory_max_index` at `buffered_raft_log.rs:710`). This isn't proving a +/// live bug in the current design (`remove_range` clamps `durable_index` +/// synchronously, so the short-circuit's precondition always holds) — it's +/// pinning down that invariant so a future change that defers the clamp +/// (e.g. copying openraft's "don't touch the watermark on truncation, rely +/// on term comparison instead") doesn't silently reopen the RPO=0 violation +/// this whole fix was for: `flush()` returning `Ok(())` before the real, +/// post-truncation tail has actually been fsynced. +/// +/// Scenario: entries 1..=100 durable. New leader truncates 81..=100 (term 2 +/// tail 81..=85 replaces it) — `durable_index` clamps to 80, +/// `memory_max_index` becomes 85. Calling `flush()` right after must NOT +/// take the short-circuit (80 < 85) — it must dispatch a real physical +/// flush for the new, not-yet-synced tail. +#[tokio::test] +async fn test_flush_does_not_short_circuit_after_truncation_regrows_the_log() { + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + "flush_no_short_circuit_after_truncation", + ); + + ctx.append_entries(1, 100, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!( + ctx.raft_log.durable_index(), + 100, + "baseline must be fully durable" + ); + + let flushes_before_truncation = flush_count.load(std::sync::atomic::Ordering::Relaxed); + + // New leader (term 2) truncates 81..=100, replaces with its own tail + // 81..=85 — durable_index clamps to 80, memory_max_index becomes 85. + let term2_tail: Vec = (81..=85).map(|i| entry(i, 2)).collect(); + ctx.raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + assert!(ctx.raft_log.durable_index() < 81, "clamp must have fired"); + assert_eq!(ctx.raft_log.last_entry_id(), 85); + + ctx.raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + ctx.drain_fsync_completions(); + + let flushes_after = flush_count.load(std::sync::atomic::Ordering::Relaxed); + assert!( + flushes_after > flushes_before_truncation, + "flush() must dispatch a real physical flush for the new tail, not \ + short-circuit on a stale-looking durable_index — before={flushes_before_truncation}, after={flushes_after}" + ); + assert_eq!( + ctx.raft_log.durable_index(), + 85, + "the new tail must actually become durable, not just claimed so" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index bf4e3996..9a3300c0 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -38,7 +38,7 @@ use crate::{FlushPolicy, RaftLog}; /// automatically — no explicit `flush()` required. #[tokio::test] async fn test_writes_become_durable_via_io_thread() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -68,6 +68,7 @@ async fn test_writes_become_durable_via_io_thread() { // Give IO thread time to process write_notify wakeup and fsync. sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); // durable_index must have advanced via IO thread auto-fsync (no explicit flush). assert_eq!( @@ -94,7 +95,7 @@ async fn test_writes_become_durable_via_io_thread() { /// N entries in one call → ≤2 fsyncs (not N), regardless of storage speed. #[tokio::test] async fn test_batch_append_produces_one_flush() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -112,6 +113,7 @@ async fn test_batch_append_produces_one_flush() { ctx.raft_log.append_entries(entries).await.unwrap(); ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 100); @@ -134,7 +136,7 @@ async fn test_batch_append_produces_one_flush() { /// `else { pending_max = 0 }` branch is skipped. /// /// ## Original bug (fixed pre-#422) -/// `handle_non_write_cmd(IOTask::Reset)` wiped the on-disk log but did NOT zero +/// `run_storage_tasks(IOTask::Reset)` wiped the on-disk log but did NOT zero /// `pending_max`. On the next `write_notify` wakeup the IO thread would compute: /// ``` /// pending_max = pending_max.max(new_end) // stale 10 wins over new 3 @@ -159,7 +161,6 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -219,7 +220,7 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() /// flush() call must be durable when flush() returns, regardless of internal batching. #[tokio::test] async fn test_flush_is_strict_durability_barrier() { - let (ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -238,6 +239,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 20, @@ -256,6 +258,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 50, @@ -289,7 +292,6 @@ async fn test_flush_propagates_io_error() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -350,7 +352,6 @@ async fn test_fsync_failure_poisons_and_rejects_writes_after_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -406,7 +407,6 @@ async fn test_replace_range_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -495,7 +495,6 @@ async fn test_purge_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -544,7 +543,6 @@ async fn test_reset_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -592,7 +590,6 @@ async fn test_save_hard_state_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -640,7 +637,6 @@ async fn test_poisoned_rejects_save_hard_state() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -665,7 +661,7 @@ async fn test_poisoned_rejects_save_hard_state() { } // ============================================================================ -// Gap fix: handle_non_write_cmd now checks is_poisoned() before executing +// Gap fix: run_storage_tasks now checks is_poisoned() before executing // ReplaceRange/Purge/Reset, instead of only checking it in run_batch_turn's // drain loop (which missed the direct-dispatch path in batch_processor's // top-level select, and the "just poisoned mid-turn" race). @@ -687,7 +683,6 @@ async fn test_poisoned_skips_replace_range() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -768,7 +763,6 @@ async fn test_poisoned_does_not_skip_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -808,7 +802,6 @@ async fn test_poisoned_skips_purge() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -818,7 +811,7 @@ async fn test_poisoned_skips_purge() { // advance_durable_and_notify() clamps against max_index — simulate a log // that already has the entry this test purges up to. - raft_log.set_max_index_for_test(1); + raft_log.set_memory_max_index_for_test(1); raft_log.poisoned.store(true, Ordering::SeqCst); let result = raft_log.purge_logs_up_to(LogId { term: 1, index: 1 }).await; @@ -890,7 +883,6 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -1001,7 +993,6 @@ async fn test_new_buffered_raft_log_starts_unpoisoned() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1032,7 +1023,6 @@ async fn test_poisoned_survives_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1070,7 +1060,6 @@ async fn test_persist_entries_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1136,7 +1125,6 @@ async fn test_poisoned_rejects_queued_persist_task() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -1202,7 +1190,6 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs index 2c4f0c02..f6602e40 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs @@ -11,7 +11,7 @@ use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_durable_index_monotonic_under_concurrency() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -39,6 +39,7 @@ async fn test_durable_index_monotonic_under_concurrency() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); // Verify monotonicity let durable = ctx.raft_log.durable_index(); @@ -124,7 +125,6 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -135,6 +135,7 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { // Arrange: entries 1..=100, all flushed — durable_index reaches 100 and // fires LogFlushed(100). simulate_insert_command(&raft_log, (1..=100).collect(), 1).await; + crate::test_utils::drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!(raft_log.durable_index(), 100); // Drain the LogFlushed(100) from the insert+flush above — not what this diff --git a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs index 95e40d8b..d3cbfab7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs @@ -22,7 +22,7 @@ use crate::{FlushPolicy, RaftLog}; /// - Expected: durable_index == 5 after explicit flush() #[tokio::test] async fn test_mem_first_entries_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -32,6 +32,7 @@ async fn test_mem_first_entries_durable_after_flush() { // Act: Append entries then wait for durability ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -50,7 +51,7 @@ async fn test_mem_first_entries_durable_after_flush() { /// - Expected: All 1000 entries durable after flush(), no data loss #[tokio::test] async fn test_mem_first_concurrent_writes_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -81,6 +82,7 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { // Wait for all entries to become durable ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -168,7 +170,7 @@ async fn test_mem_first_buffers_entries_before_flush() { /// - Expected: Entries become durable after flush #[tokio::test] async fn test_mem_first_flushes_asynchronously() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -181,6 +183,7 @@ async fn test_mem_first_flushes_asynchronously() { // Act: Explicit flush ctx.raft_log.flush().await.unwrap(); sleep(Duration::from_millis(100)).await; // Allow async flush + ctx.drain_fsync_completions(); // Assert: Entries now durable assert!( @@ -236,7 +239,7 @@ async fn test_mem_first_concurrent_buffering() { /// - Expected: Flush triggered at threshold #[tokio::test] async fn test_batched_flushes_at_threshold() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 10000, // High interval to test threshold trigger }, @@ -246,6 +249,7 @@ async fn test_batched_flushes_at_threshold() { // Act: Append exactly threshold entries ctx.append_entries(1, 5, 1).await; sleep(Duration::from_millis(100)).await; // Allow flush + ctx.drain_fsync_completions(); // Assert: Entries should be flushed assert!( @@ -261,7 +265,7 @@ async fn test_batched_flushes_at_threshold() { /// - Expected: Flush triggered by timer #[tokio::test] async fn test_batched_flushes_at_interval() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -271,6 +275,7 @@ async fn test_batched_flushes_at_interval() { // Act: Append few entries and wait for interval ctx.append_entries(1, 2, 1).await; sleep(Duration::from_millis(200)).await; // Wait for interval flush + ctx.drain_fsync_completions(); // Assert: Entries flushed by timer assert!( diff --git a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs index 508c3bd5..ab405f0c 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs @@ -22,7 +22,6 @@ fn setup_memory() -> Arc> { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs index 52826bfb..62bc61ff 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs @@ -59,7 +59,6 @@ async fn test_reset_performance_during_active_flush() { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -119,7 +118,6 @@ async fn test_filter_conflicts_performance_during_flush() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -201,7 +199,6 @@ async fn test_fresh_cluster_performance_consistency() { let config = PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs index 95f49106..6efc9e95 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -45,7 +45,7 @@ fn entry( /// claim durability for an index that doesn't exist in the log anymore. #[tokio::test] async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // isolate from the safety-net timer }, @@ -77,6 +77,7 @@ async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { // Trigger a disk sync and give it time to complete. ctx.raft_log.flush().await.unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); // The follower must never advertise durability for an index it doesn't // actually have. If persisted_index wasn't clamped down during the @@ -133,7 +134,6 @@ async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index a0d689fa..d9e9876a 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -474,7 +474,6 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs index 1a206f84..2f495a53 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -51,7 +51,6 @@ async fn test_append_entries_waits_for_storage_engine_before_returning() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage.clone()), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs index 36b88ce3..4d45b7ac 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs @@ -82,7 +82,6 @@ async fn test_quorum_uses_durable_index_not_last_entry_id() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -132,7 +131,6 @@ async fn test_election_eligibility_reads_memory_log_not_durable_index() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -165,7 +163,7 @@ async fn test_election_eligibility_reads_memory_log_not_durable_index() { /// that a separate, broader safety argument for #446 relies on. #[tokio::test] async fn test_majority_matched_index_requires_actual_majority_of_reports() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -174,6 +172,7 @@ async fn test_majority_matched_index_requires_actual_majority_of_reports() { ctx.append_entries(1, 10, 1).await; ctx.raft_log.flush().await.unwrap(); // leader's own entries now durable through 10 + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 10); @@ -198,7 +197,7 @@ async fn test_majority_matched_index_requires_actual_majority_of_reports() { /// quorum should proceed normally. #[tokio::test] async fn test_quorum_succeeds_after_leader_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 999_999, // only threshold trigger, no timer }, @@ -210,6 +209,7 @@ async fn test_quorum_succeeds_after_leader_flush() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.last_entry_id(), 1); assert_eq!( @@ -249,7 +249,6 @@ async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -274,7 +273,7 @@ async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { /// After explicit flush, durable_index must equal last_entry_id. #[tokio::test] async fn test_durable_index_equals_last_entry_id_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -283,6 +282,7 @@ async fn test_durable_index_equals_last_entry_id_after_flush() { ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); let last = ctx.raft_log.last_entry_id(); let durable = ctx.raft_log.durable_index(); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs index 373aeffd..bbec2429 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs @@ -44,7 +44,7 @@ async fn test_log_matching_property() { #[tokio::test] async fn test_leader_completeness_property() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,6 +54,7 @@ async fn test_leader_completeness_property() { // Leader writes entries and flushes so durable_index = 10 ctx.append_entries(1, 10, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Simulate majority replication scenario: // Leader has: [1,2,3,4,5,6,7,8,9,10], durable_index=10 @@ -81,7 +82,7 @@ async fn test_leader_completeness_property() { #[tokio::test] async fn test_calculate_majority_matched_index_case0() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -94,6 +95,7 @@ async fn test_calculate_majority_matched_index_case0() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2, 3], 2).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), @@ -127,7 +129,7 @@ async fn test_calculate_majority_matched_index_case1() { #[tokio::test] async fn test_calculate_majority_matched_index_case2() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -143,6 +145,7 @@ async fn test_calculate_majority_matched_index_case2() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2], 2).await; simulate_insert_command(&ctx.raft_log, vec![3], 3).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), ctx.raft_log.calculate_majority_matched_index(ct, ci, vec![4, 2]) @@ -196,7 +199,7 @@ async fn test_calculate_majority_matched_index_case4() { #[tokio::test] async fn test_calculate_majority_matched_index_case5() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -215,6 +218,7 @@ async fn test_calculate_majority_matched_index_case5() { let raft_log_entry_ids: Vec = (1..=raft_log_length).collect(); simulate_insert_command(&ctx.raft_log, raft_log_entry_ids, 1).await; + ctx.drain_fsync_completions(); assert_eq!( Some(peer2_match), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs index 5f0358c7..c6259867 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -1,7 +1,7 @@ //! `IOTask::ReplaceRange` (term-conflict truncation, see //! `filter_out_conflicts_and_append`'s slow path) writes to the storage engine //! synchronously and bumps `pending_max`, but is dispatched through the -//! `receiver.recv()` => `cmd => { handle_non_write_cmd(...) }` arm of the IO +//! `receiver.recv()` => `cmd => { run_storage_tasks(...) }` arm of the IO //! thread's select loop — a branch that, unlike `run_batch_turn`, never calls //! `fsync_coordinator.submit()`. If no further `append_entries()` call arrives //! afterward (which would separately trigger a `run_batch_turn` via @@ -44,7 +44,7 @@ fn entry( /// truncation, because the replaced entries' fsync was never submitted. #[tokio::test] async fn test_replace_range_becomes_durable_without_a_following_append() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe }, @@ -54,6 +54,7 @@ async fn test_replace_range_becomes_durable_without_a_following_append() { // Arrange: log [1,2,3] all term=1, explicitly flushed durable. ctx.append_entries(1, 3, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 3, "baseline must be durable"); // Act: leader (term=2) sends entries that conflict at index=2 and extend @@ -76,6 +77,7 @@ async fn test_replace_range_becomes_durable_without_a_following_append() { // Give the IO thread ample time to have submitted fsync, if anything // besides the (disabled) safety net were going to do it. tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); // FIXED: ReplaceRange's handler now submits fsync directly instead of // relying on a following append/notify or the safety net. diff --git a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs index ad909049..79ebce89 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs @@ -50,7 +50,6 @@ fn test_io_thread_survives_runtime_drop() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 50, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -159,7 +158,6 @@ async fn test_shutdown_handles_slow_workers() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -287,7 +285,6 @@ async fn test_replace_range_failure_propagates_error_and_shuts_down_io_thread() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // no auto-flush }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs index 16714b50..6e5fe913 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -54,7 +54,6 @@ async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 603a0c01..93e05145 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -1,6 +1,7 @@ use crate::BufferedRaftLog; use crate::Error; use crate::LogStore; +use crate::RaftLog; use crate::Result; use crate::TypeConfig; use std::sync::Arc; @@ -9,20 +10,15 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::oneshot; use tracing::error; -/// Tracks whether a fsync task is currently running on the blocking pool. -/// Ensures at most one physical `flush_wal` call is in flight at any time, -/// restoring natural batching: entries that arrive while a fsync is running -/// accumulate in `pending_max`/`pending_replies`, and are picked up by the -/// SAME task once it finishes its current round — rather than spawning a -/// new competing task per `write_notify` wakeup. +/// Schedules physical fsync calls — batches concurrent requests into one +/// flush() at a time. Does not judge whether results are still valid; see +/// BufferedRaftLog::apply_durable_report. pub(super) struct FsyncCoordinator { inflight: AtomicBool, pending_max: AtomicU64, pending_replies: Mutex>>>, - // Fencing token (like Raft's `term`) for in-flight fsync results. Private — - // only bump via a fence_*() verb below, one per invalidating event. Never a - // value-passing variant (index math can under-fence, see fence_truncation()). + // Lets a stale round skip its reply early. Optional — not required for correctness. generation: AtomicU64, } @@ -78,6 +74,7 @@ impl FsyncCoordinator { let gen_at_start = self.generation.load(Ordering::Acquire); let max_index = self.pending_max.swap(0, Ordering::AcqRel); + let max_term = this.entry_term(max_index).unwrap_or(0); let replies = std::mem::take(&mut *self.pending_replies.lock().unwrap()); if this.is_poisoned() { @@ -126,8 +123,7 @@ impl FsyncCoordinator { r }; - // Fence check: if a reset happened while this batch was in flight, - // its result is for data that no longer exists — discard. + // Skip replying if this round is already known stale. if self.generation.load(Ordering::Acquire) != gen_at_start { for reply in replies { let _ = reply.send(Err(crate::Error::Fatal( @@ -138,7 +134,7 @@ impl FsyncCoordinator { } match &result { - Ok(()) => this.advance_durable_and_notify(max_index), + Ok(()) => this.notify_fsync_completed(max_index, max_term), Err(e) => { // One fsync failure = fatal, no threshold, no retry-and-hope. // Durability state is now unknown, this node @@ -180,13 +176,6 @@ impl FsyncCoordinator { self.bump_generation(); } - /// Called from `remove_range()` before a truncation is applied. Bumps - /// `generation` to fence any fsync already in flight for data this - /// truncation is about to discard — mirrors `fence_reset()`, but does - /// NOT touch `pending_max`/`pending_replies`: unlike a full reset, - /// a truncation's own `IOTask::ReplaceRange` handler submits a fresh, - /// correct `max_index` for the surviving log right after this runs, - /// so there is nothing stale left to drain. pub(super) fn fence_truncation( &self, new_max: u64, diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 3b2a371e..93fdf8c0 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -14,13 +14,16 @@ use super::*; use crate::FlushPolicy; +use crate::InternalEvent; use crate::MockLogStore; use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; use crate::Result; +use d_engine_proto::common::Entry; use std::sync::Arc; +use tokio::sync::mpsc; /// Build a `BufferedRaftLog` for direct `FsyncCoordinator` method calls — /// never `.start()`-ed, no IO thread, no channel plumbing. Only `log_store`/ @@ -33,7 +36,6 @@ fn minimal_raft_log(storage: MockStorageEngine) -> Arc::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); coord.pending_max.store(5, Ordering::Release); coord.run_until_caught_up(&raft_log); + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(index, term); + } + assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 5, @@ -480,9 +511,31 @@ fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { "run_until_caught_up_accepts_result_when_generation_unchanged".into(), ); let coord = FsyncCoordinator::new(); - let raft_log = minimal_raft_log(storage); - // advance_durable_and_notify() clamps against max_index — matches pending_max below. - raft_log.set_max_index_for_test(5); + // See test_run_until_caught_up_advances_durable_index_on_success for why + // this doesn't use minimal_raft_log(): needs a real backing entry for + // try_advance_durable_index's content check, and a registered + // log_flush_tx to receive the FsyncCompleted report. + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); // Two unrelated fences happened earlier — generation is 2, not 0 — before // this round is even recorded as in flight. @@ -498,6 +551,11 @@ fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { // Nothing fences this round while it runs — generation stays at 2. coord.run_until_caught_up(&raft_log); + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(index, term); + } + assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 5, @@ -525,7 +583,7 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); // advance_durable_and_notify() clamps against max_index — matches pending_max below. - raft_log.set_max_index_for_test(10); + raft_log.set_memory_max_index_for_test(10); // Simulate two submit() calls that both lost the CAS while a round was // in flight — both just accumulated into the same pending state. diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index 0b9b1496..c5560be8 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -77,6 +77,18 @@ pub trait RaftLog: Send + Sync + 'static { /// - DiskFirst: equals `last_entry_id()` (every append blocks until durable). fn durable_index(&self) -> u64; + /// Content-validated durable-watermark advance. `index`/`term` describe + /// what a completed fsync claims is now safe — rejected (`None`) if + /// `entry_term(index) != Some(term)`, meaning the log content at that + /// index has changed (truncated + replaced) since fsync started on it. + /// `Some(new_value)` only when it actually advanced — callers use this + /// to decide whether to fire `handle_log_flushed`. + fn try_advance_durable_index( + &self, + index: u64, + term: u64, + ) -> Option; + /// Returns the LogId (term + index) of the last entry. /// /// # Returns diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index 43e955e9..a9376c82 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -21,6 +21,7 @@ pub struct BufferedRaftLogTestContext { pub storage: Arc, pub flush_policy: FlushPolicy, pub instance_id: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl BufferedRaftLogTestContext { @@ -35,12 +36,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -50,9 +51,21 @@ impl BufferedRaftLogTestContext { storage, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, } } + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only unit tests. Call + /// after any operation that should make `durable_index` advance + /// (`append_entries`, `flush`, truncation + resync, ...) and before + /// asserting on `durable_index()` — see + /// `drain_and_apply_fsync_completions` for why this is necessary since + /// #446/#447. + pub fn drain_fsync_completions(&mut self) { + drain_and_apply_fsync_completions(&self.raft_log, &mut self.log_flush_rx); + } + /// Helper to append a batch of entries with specified range and term pub async fn append_entries( &self, @@ -88,12 +101,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(std::time::Duration::from_millis(10)); let ctx = Self { @@ -101,6 +114,7 @@ impl BufferedRaftLogTestContext { storage, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, }; (ctx, flush_count) } @@ -114,12 +128,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: self.flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -129,6 +143,7 @@ impl BufferedRaftLogTestContext { storage, flush_policy: self.flush_policy.clone(), instance_id: self.instance_id.clone(), + log_flush_rx, } } } @@ -228,3 +243,24 @@ pub async fn simulate_delete_command( raft_log.insert_batch(entries).await.unwrap(); raft_log.flush().await.unwrap(); } + +/// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, which +/// `BufferedRaftLog`-only unit tests don't have running. Since #446/#447, +/// `durable_index` only advances when something calls +/// `try_advance_durable_index(index, term)` in response to that event — +/// `FsyncCoordinator`/`IOTask::ReplaceRange`'s `notify_fsync_completed` only +/// *sends* the event, it never writes `durable_index` itself. A test that +/// registers a `log_flush_tx` and wants to see `durable_index()` actually +/// advance must drain that channel through this helper — otherwise the +/// event sits unread and `durable_index()` never moves, no matter how long +/// you sleep. +pub fn drain_and_apply_fsync_completions( + raft_log: &Arc>, + log_flush_rx: &mut tokio::sync::mpsc::UnboundedReceiver, +) { + while let Ok(event) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted { index, term } = event { + raft_log.try_advance_durable_index(index, term); + } + } +} diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index 7990cf58..b8a32bdd 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -322,7 +322,7 @@ impl MockStorageEngine { /// After a failed fsync, `batch_processor` logs the error and does NOT zero /// `pending_max` (the success branch `else { pending_max = 0 }` is not taken). /// This is the deterministic pre-condition needed to exercise the bug where - /// `handle_non_write_cmd(IOTask::Reset)` forgets to zero `pending_max`. + /// `run_storage_tasks(IOTask::Reset)` forgets to zero `pending_max`. pub fn not_durable_first_flush_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); let mut mock_meta_store = MockMetaStore::new(); @@ -393,7 +393,7 @@ impl MockStorageEngine { /// Create a MockStorageEngine where `replace_range()` always fails, /// simulating a fatal storage error during conflict-resolution - /// (truncate + write). `handle_non_write_cmd`'s `IOTask::ReplaceRange` + /// (truncate + write). `run_storage_tasks`'s `IOTask::ReplaceRange` /// arm treats this as unrecoverable — disk state is now uncertain. pub fn not_durable_replace_range_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 0556b0e2..eb675479 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -60,7 +60,6 @@ async fn test_set_raft_log_replaces_default() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, mock_storage_engine.clone(), diff --git a/d-engine-server/src/test_utils/integration/mod.rs b/d-engine-server/src/test_utils/integration/mod.rs index 00365525..e0208b75 100644 --- a/d-engine-server/src/test_utils/integration/mod.rs +++ b/d-engine-server/src/test_utils/integration/mod.rs @@ -183,7 +183,6 @@ pub fn setup_raft_components( flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage_engine.clone(), diff --git a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs index ef0f53e2..9130f08f 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs @@ -61,7 +61,7 @@ async fn test_crash_recovery() { #[tokio::test] async fn test_crash_recovery_with_multiple_entries() { // Create and populate storage - let original_ctx = TestContext::new( + let mut original_ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -85,6 +85,7 @@ async fn test_crash_recovery_with_multiple_entries() { // Ensure all entries are persisted for DiskFirst strategy original_ctx.raft_log.flush().await.unwrap(); + original_ctx.drain_fsync_completions(); // Verify all entries are in memory and durable assert_eq!(original_ctx.raft_log.durable_index(), 5); @@ -127,7 +128,6 @@ async fn test_partial_flush_with_graceful_shutdown() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -163,7 +163,6 @@ async fn test_partial_flush_with_graceful_shutdown() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -207,7 +206,6 @@ async fn test_partial_flush_after_crash() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -255,7 +253,6 @@ async fn test_partial_flush_after_crash() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -381,7 +378,6 @@ async fn test_memfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -414,7 +410,6 @@ async fn test_diskfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -447,7 +442,6 @@ async fn test_diskfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index 8687f2a4..35b4300c 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -33,6 +33,7 @@ pub struct TestContext { pub _temp_dir: Option, pub flush_policy: FlushPolicy, pub path: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl TestContext { @@ -49,12 +50,12 @@ impl TestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(Duration::from_millis(10)); @@ -65,6 +66,23 @@ impl TestContext { storage, flush_policy, _temp_dir: Some(temp_dir), + log_flush_rx, + } + } + + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only integration + /// tests. Since #446/#447, `durable_index` only advances when something + /// drains that event and calls `try_advance_durable_index` — call this + /// after any operation that should make `durable_index` advance and + /// before asserting on it. Not needed after `recover_from_crash()`: the + /// recovered context's `durable_index` is derived directly from on-disk + /// state at construction, not from this event. + pub fn drain_fsync_completions(&mut self) { + while let Ok(event) = self.log_flush_rx.try_recv() { + if let d_engine_core::InternalEvent::FsyncCompleted { index, term } = event { + self.raft_log.try_advance_durable_index(index, term); + } } } @@ -88,12 +106,12 @@ impl TestContext { 1, PersistenceConfig { flush_policy: self.flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); @@ -103,6 +121,7 @@ impl TestContext { flush_policy: self.flush_policy.clone(), _temp_dir: Some(temp_dir), path: self.path.clone(), + log_flush_rx, } } diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 1d050f53..3e7b81c7 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -34,7 +34,6 @@ mod filter_out_conflicts_and_append_performance_tests { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; @@ -104,7 +103,6 @@ mod filter_out_conflicts_and_append_performance_tests { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; diff --git a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs index 351e5a8d..ea60cdc1 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs @@ -44,7 +44,7 @@ use d_engine_core::RaftLog; #[tokio::test] async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,6 +54,7 @@ async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { // First 5 entries, explicitly flushed: genuinely durable, deterministic. ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 5); // 3-node cluster: both followers already report match_index=5 (post-Stage2 @@ -81,6 +82,7 @@ async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { // simulated crash — keeps this test's crash/recovery assertions exact, not bounded. ctx.append_entries(6, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 10); let recovered = ctx.recover_from_crash(); diff --git a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs index f9c9dfe6..acf79449 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs @@ -13,7 +13,7 @@ use super::TestContext; #[tokio::test] async fn test_log_compaction() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -23,6 +23,7 @@ async fn test_log_compaction() { // With MemFirst, entries are buffered and flushed asynchronously. // Wait for all entries to become durable before checking durable_index. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Compact first 50 entries ctx.raft_log.purge_logs_up_to(LogId { index: 50, term: 1 }).await.unwrap(); diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index 3f41a4c0..7968af39 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -22,7 +22,7 @@ use super::TestContext; #[tokio::test] async fn test_high_concurrency() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,6 +52,7 @@ async fn test_high_concurrency() { // With MemFirst, entries are buffered; wait for all to be durable before asserting. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify all entries persisted assert_eq!(ctx.raft_log.durable_index(), 1000); @@ -154,7 +155,7 @@ mod mem_first_tests { #[tokio::test] async fn test_async_persistence() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -164,6 +165,7 @@ mod mem_first_tests { // Trigger flush ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify persistence assert_eq!(ctx.raft_log.durable_index(), 100); diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index b5432c04..b5ab1764 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -60,7 +60,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -max_buffered_entries = 10000 ``` **Key differences from single-node expansion:** diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index d036239d..1d961cb3 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -44,8 +44,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/single-node-expansion/config/n2.toml b/examples/single-node-expansion/config/n2.toml index bad8ce26..0e4bb985 100644 --- a/examples/single-node-expansion/config/n2.toml +++ b/examples/single-node-expansion/config/n2.toml @@ -29,7 +29,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/single-node-expansion/config/n3.toml b/examples/single-node-expansion/config/n3.toml index 67cf1fd2..32dc4c3b 100644 --- a/examples/single-node-expansion/config/n3.toml +++ b/examples/single-node-expansion/config/n3.toml @@ -31,7 +31,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index bb6db45a..1acb62d9 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 95280d76..959b904a 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 4aa67b3b..12c0ff72 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/docker/config/n1.toml b/examples/three-nodes-standalone/docker/config/n1.toml index 8deff325..a5b007dc 100644 --- a/examples/three-nodes-standalone/docker/config/n1.toml +++ b/examples/three-nodes-standalone/docker/config/n1.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n2.toml b/examples/three-nodes-standalone/docker/config/n2.toml index 7e7eb644..06c54f50 100644 --- a/examples/three-nodes-standalone/docker/config/n2.toml +++ b/examples/three-nodes-standalone/docker/config/n2.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n3.toml b/examples/three-nodes-standalone/docker/config/n3.toml index 73d6bc59..2cfe4ad9 100644 --- a/examples/three-nodes-standalone/docker/config/n3.toml +++ b/examples/three-nodes-standalone/docker/config/n3.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false