From e2ebdb2e56899a8b02d4fb27de172aa385bc16ac Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Tue, 6 Oct 2026 01:40:01 -0600 Subject: [PATCH 01/19] test: migrate fault transport to owned Groupnet buffers --- tests/it/common/faultnet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/it/common/faultnet.rs b/tests/it/common/faultnet.rs index 6f82d59..ce5c602 100644 --- a/tests/it/common/faultnet.rs +++ b/tests/it/common/faultnet.rs @@ -138,7 +138,7 @@ impl Transport for FaultTransport { if let Some((tx, delay)) = self.net.route(&self.id, to) { let inbound = Inbound { from: self.id.clone(), - msg: msg.to_vec(), + msg: msg.to_vec().into(), }; if delay.is_zero() { let _ = tx.send(inbound); From 8d5cc2f39ede1526d5e15f0b9a97bfca84aa36b3 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:13:57 -0600 Subject: [PATCH 02/19] feat(sync): managed Groupnet links, explicit gossip neighbors, and split coherence and test modules Import the unfinished work from the local clone as it stood: the gossip node starts one managed network over an explicitly admitting UDP link, the chart lists every ordinal as a neighbor, the planned stop closes the network after the seal, the apply loop moves to sync/coherence/apply.rs, the sync test fixtures move to sync/tests/fixtures.rs, and the strong_rw workload moves to tests/it/strong_rw/workload.rs. Groupnet builds without default features, with the memory binding for tests only. It was written against a Groupnet revision between fb97452 and 30407c0 and does not build at either; the following commits port it. --- AGENTS.md | 5 + Cargo.toml | 4 +- README.md | 11 +- .../helm/s3cache/templates/statefulset.yaml | 21 +- deploy/helm/s3cache/values.yaml | 8 +- src/cache/stop.rs | 3 + src/cache/tests.rs | 94 ++- src/sync/coherence.rs | 156 +--- src/sync/coherence/apply.rs | 142 ++++ src/sync/config.rs | 56 +- src/sync/stop.rs | 8 + src/sync/tests.rs | 111 +-- src/sync/tests/fixtures.rs | 76 ++ src/sync/volatile.rs | 30 +- tests/it/common/fleet.rs | 23 +- tests/it/strong_rw.rs | 678 +----------------- tests/it/strong_rw/workload.rs | 662 +++++++++++++++++ 17 files changed, 1133 insertions(+), 955 deletions(-) create mode 100644 src/sync/coherence/apply.rs create mode 100644 src/sync/tests/fixtures.rs create mode 100644 tests/it/strong_rw/workload.rs diff --git a/AGENTS.md b/AGENTS.md index c2d7909..fc4bf97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -163,6 +163,11 @@ documentation-only edit does not require an image build. | `S3CACHE_STATS_SECS` | `60` | Stats log interval | | `S3CACHE_METRICS_LISTEN` | empty (disabled) | Bind address for the Prometheus text endpoint (`GET /metrics`) | +Gossip uses one managed network: `S3CACHE_GOSSIP_SEEDS` explicitly admits UDP routing +neighbors on both endpoints; DNS and advertisements refresh addresses only. The chart +lists every ordinal. Startup awaits link binding, and planned shutdown closes the +links and resolver after mutation drain and feed seal. + `GET /ready` on the metrics listener is the pod readiness probe (`CachingProxy::probe_ready`): ready once this node's index is complete, or while no live (`Alive`/`Suspect`) peer declares the `s3cache:indexed` gossip capability or diff --git a/Cargo.toml b/Cargo.toml index 339935d..9f9e639 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ blake3 = "1.8.5" bytes = { version = "1", features = ["serde"] } futures = "0.3" getrandom = "0.4" -groupnet = { git = "https://github.com/napbat/groupnet", rev = "fb97452387e396c9b77e91f15c654c69489e2378", features = ["consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } +groupnet = { git = "https://github.com/napbat/groupnet", rev = "fb97452387e396c9b77e91f15c654c69489e2378", default-features = false, features = ["runtime", "consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } http = "1" http-body-util = "0.1" # Already under hyper-util/s3s; direct so the metrics endpoint can build its own responses. @@ -106,6 +106,8 @@ debug = "line-tables-only" opt-level = 2 [dev-dependencies] +# Managed in-process control links are test-only; production binds UDP. +groupnet = { workspace = true, features = ["mem"] } # Real bincode 1 bytes: the fixtures that prove `codec::legacy` still reads what # earlier releases stored. Never a runtime dependency. bincode.workspace = true diff --git a/README.md b/README.md index fba4cd9..11a393e 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,13 @@ The fast path stays in local memory (LIST is served from RAM; nothing remote is read path). Loss is **detected, never silent**: a peer that falls behind the feed's ring, or a peer restart, surfaces as a *gap* — the node distrusts every local body and resyncs its index from the origin, which is the authority the index caches, so a copy is served -again only once that index has proved it current. Only seeds need static addressing; -every other peer resolves through gossiped advertisements. The ring is bounded by both +again only once that index has proved it current. `S3CACHE_GOSSIP_SEEDS` explicitly +admits routing neighbors: configure both endpoints of every intended link. DNS and +gossiped advertisements refresh admitted neighbors' addresses; discovering another +member does not grant admission. The Helm chart configures every ordinal, including +later-starting peers. Groupnet starts one managed network asynchronously and closes +its links and resolver after the planned mutation drain and feed seal. +The ring is bounded by both event count and encoded bytes: sustained long-key writes shorten its replay window and surface the same explicit gap instead of growing one state entry past the transport envelope and silently arresting peer application. @@ -507,7 +512,7 @@ Unrelated documentation-only edits do not require a container build. | `S3CACHE_DISK_CACHE` | (empty) | Directory for the warm disk tier; unset = no disk tier | | `S3CACHE_DISK_CACHE_BYTES` | `10737418240` (10 GB) | Warm disk tier capacity (bytes) | | `S3CACHE_GOSSIP_BIND` | (empty) | UDP bind for the gossip write feed (e.g. `0.0.0.0:7946`); unset = single-node, no coherence layer | -| `S3CACHE_GOSSIP_SEEDS` | (empty) | Comma-separated seed peers as `id=host:port`; only seeds need static addressing | +| `S3CACHE_GOSSIP_SEEDS` | (empty) | Explicit UDP routing neighbors as comma-separated `id=host:port` pairs; configure both endpoints (self entries are ignored) | | `S3CACHE_GOSSIP_ADVERTISE` | bind addr | Address peers should dial back (set under NAT/container networking) | | `S3CACHE_CONSISTENCY` | `strong` | `strong`: lease-backed — a node serves locally only while it holds a serve-lease, and a write ends at an ack or at a lapse. `strong-acks`: the pre-lease ack-only mechanism, **deprecated**, removed next release. `bounded`: ~one-hop freshness, no per-write ack round (large clusters; set uniformly) | | `S3CACHE_LEASE_MS` | `2000` | Coherence-lease duration `D` (`strong` only): the writer's worst-case stall on a silent peer, each reader's window between confirmations, and the fleet's membership `dead_timeout`. Renewal traffic is one small entry per node per `D/3` | diff --git a/deploy/helm/s3cache/templates/statefulset.yaml b/deploy/helm/s3cache/templates/statefulset.yaml index a783a24..c924cda 100644 --- a/deploy/helm/s3cache/templates/statefulset.yaml +++ b/deploy/helm/s3cache/templates/statefulset.yaml @@ -90,11 +90,10 @@ spec: - name: S3CACHE_METRICS_LISTEN value: "0.0.0.0:{{ .Values.metrics.port | int64 }}" {{- end }} - # Pod name = gossip node id (HOSTNAME is set by the kubelet). Pod-0 is the - # only static seed; the rest resolves via gossiped advertisements. The - # advertise address must be the routable pod IP (peers parse it as an - # address), hence the downward API rather than a DNS name. Always on: a - # single replica is a cluster of one, and scaling is just replicaCount. + # Pod name = gossip node id (HOSTNAME is set by the kubelet). Every + # ordinal explicitly admits the other routing neighbors below. The + # advertise address is the routable pod IP, supplied by the downward API. + # Always on: a single replica is a cluster of one; scaling is replicaCount. - name: POD_IP valueFrom: fieldRef: @@ -103,12 +102,14 @@ spec: value: "0.0.0.0:{{ .Values.gossip.port }}" - name: S3CACHE_GOSSIP_ADVERTISE value: "$(POD_IP):{{ .Values.gossip.port }}" - # Two seed ordinals when there can be a peer: each pod skips itself, - # leaving pod-0 seeding pod-1 and everyone else seeding pod-0. A - # single-replica cluster names only pod-0, so its skipped self entry - # does not generate a permanent unresolved-peer warning. + # Explicitly admit every ordinal, including peers that have not started. + # Each pod skips itself; a singleton therefore has no unresolved seed. + {{- $gossipPeers := list }} + {{- range $ordinal := until (int .Values.replicaCount) }} + {{- $gossipPeers = append $gossipPeers (printf "s3cache-%d=s3cache-%d.s3cache-gossip:%d" $ordinal $ordinal (int $.Values.gossip.port)) }} + {{- end }} - name: S3CACHE_GOSSIP_SEEDS - value: {{ if gt (int .Values.replicaCount) 1 }}"s3cache-0=s3cache-0.s3cache-gossip:{{ .Values.gossip.port }},s3cache-1=s3cache-1.s3cache-gossip:{{ .Values.gossip.port }}"{{ else }}"s3cache-0=s3cache-0.s3cache-gossip:{{ .Values.gossip.port }}"{{ end }} + value: {{ join "," $gossipPeers | quote }} # Peer index bootstrap: a joining pod copies a ready peer's verified index # instead of listing the origin. Every ordinal shares this StatefulSet's # origin identity and bucket inventory; with no configured buckets there diff --git a/deploy/helm/s3cache/values.yaml b/deploy/helm/s3cache/values.yaml index d249115..e42aa0f 100644 --- a/deploy/helm/s3cache/values.yaml +++ b/deploy/helm/s3cache/values.yaml @@ -81,10 +81,10 @@ diskCache: volume: # any pod-volume spec backing diskCache.path emptyDir: {} -# Cross-node coherence via the gossip write feed (groupnet) — always on, no extra -# services, no toggle: pods gossip over a headless UDP service, pod-0 is the only -# static seed (everyone else resolves through gossiped advertisements), and pod names -# are the node identities. A single replica is a cluster of one. +# Cross-node coherence via managed gossip links (groupnet) — always on, no extra +# services, no toggle: each pod explicitly admits every ordinal over the headless +# UDP service. DNS and gossip refresh addresses, never neighbor admission. Pod names +# are node identities; a single replica is a cluster of one. gossip: port: 7946 # strong (default): coherence leases. A pod may answer from its own index or body cache diff --git a/src/cache/stop.rs b/src/cache/stop.rs index 88d03f5..cc88fe1 100644 --- a/src/cache/stop.rs +++ b/src/cache/stop.rs @@ -196,6 +196,9 @@ impl CachingProxy { async { tokio::time::timeout(DRAIN_WAIT, drain).await.is_ok() }, self.seal_writes(SEAL_WAIT), ); + if let Some(sync) = &self.sync { + sync.close().await; + } if drained { info!("graceful shutdown complete"); } else { diff --git a/src/cache/tests.rs b/src/cache/tests.rs index 877ec37..a4af8be 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -5,8 +5,10 @@ use std::time::{Duration, SystemTime}; use bytes::Bytes; use groupnet::consistency::{LeaseConfig, WriteFeed, advertised_head}; use groupnet::core::{Config, NodeId}; +use groupnet::network::RouterConfig; use groupnet::runtime::Node; -use groupnet::transport::mem::{MemTransport, Network}; +use groupnet::transport::link::{BoundLink, LinkConfig}; +use groupnet::transport::mem::Network; use http::HeaderMap; use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, ListObjectsV2Input, Timestamp}; use s3s::{S3, S3ErrorCode, S3Request}; @@ -38,12 +40,18 @@ fn brisk() -> Config { /// One leased node, alone in its group. A solo reader's granter roster is empty, so /// its lease confirms vacuously — which is exactly why the shell's warm-up guard, /// not the confirmation, is what a booting node has to get past. -fn solo(id: &str) -> (Node, Arc) { +async fn solo(id: &str) -> (Node, Arc) { let net = Network::new(); let me = NodeId::new(id); - let node = Node::builder(me.clone(), net.endpoint(me.clone())) + let node = Node::builder(me.clone()) + .link(BoundLink::new( + net.endpoint(me.clone()), + LinkConfig::new(Vec::new()), + )) .config(brisk()) - .spawn(); + .start() + .await + .expect("solo managed node starts"); let group = node.join_group("s3cache"); let sync = WriteSync::attach( group, @@ -123,7 +131,7 @@ async fn revoke_before_local_return( #[tokio::test] async fn a_lease_revoked_after_local_body_lookup_forces_origin_get() { - let (_node, sync) = solo("return-fence-get"); + let (_node, sync) = solo("return-fence-get").await; let proxy = proxy(Some(sync.clone())); let body = cached("v1", at(1_700_000_000)); body.mark_trusted(proxy.obj_cache.suspect_gen()); @@ -158,7 +166,7 @@ async fn a_lease_revoked_after_local_body_lookup_forces_origin_get() { #[tokio::test] async fn a_lease_revoked_after_index_answer_forces_origin_list() { - let (_node, sync) = solo("return-fence-list"); + let (_node, sync) = solo("return-fence-list").await; let proxy = proxy(Some(sync.clone())); index(&proxy, "k", Some("v1"), at(1_700_000_000)); synced(&proxy); @@ -294,7 +302,7 @@ fn counter(proxy: &CachingProxy, name: &str) -> u64 { /// body of that same version can pass revalidation after warm-up completes. #[tokio::test] async fn an_unknown_read_observation_keeps_the_origin_mtime() { - let (_node, sync) = solo("observed-origin-mtime"); + let (_node, sync) = solo("observed-origin-mtime").await; let proxy = proxy(Some(sync)); let modified = at(1_700_000_000); let observed = ObservedObject { @@ -336,14 +344,32 @@ async fn a_freshness_timeout_routes_to_origin_and_is_counted() { let net = Network::new(); let a_id = NodeId::new("barrier-a"); let b_id = NodeId::new("barrier-b"); - let a_node = Node::builder(a_id.clone(), net.endpoint(a_id.clone())) - .seed(b_id.clone()) + let a_node = Node::builder(a_id.clone()) + .link(BoundLink::new( + net.endpoint(a_id.clone()), + LinkConfig::new(vec![b_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); - let b_node = Node::builder(b_id.clone(), net.endpoint(b_id.clone())) - .seed(a_id.clone()) + .start() + .await + .expect("A managed node starts"); + let b_node = Node::builder(b_id.clone()) + .link(BoundLink::new( + net.endpoint(b_id.clone()), + LinkConfig::new(vec![a_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); + .start() + .await + .expect("B managed node starts"); let a_group = a_node.join_group("s3cache"); let b_group = b_node.join_group("s3cache"); let sync = Arc::new(WriteSync::attach( @@ -404,14 +430,32 @@ async fn a_stalled_write_is_a_503_not_a_success() { let net = Network::new(); let a_id = NodeId::new("stall-a"); let b_id = NodeId::new("stall-b"); - let a_node = Node::builder(a_id.clone(), net.endpoint(a_id.clone())) - .seed(b_id.clone()) + let a_node = Node::builder(a_id.clone()) + .link(BoundLink::new( + net.endpoint(a_id.clone()), + LinkConfig::new(vec![b_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); - let b_node = Node::builder(b_id.clone(), net.endpoint(b_id.clone())) - .seed(a_id.clone()) + .start() + .await + .expect("A managed node starts"); + let b_node = Node::builder(b_id.clone()) + .link(BoundLink::new( + net.endpoint(b_id.clone()), + LinkConfig::new(vec![a_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); + .start() + .await + .expect("B managed node starts"); let lease = LeaseConfig::for_duration(Duration::from_millis(300)); // A advertises the ack tier but runs no apply loop, so it never acknowledges. let _peer = WriteSync::attach( @@ -456,7 +500,7 @@ async fn a_stalled_write_is_a_503_not_a_success() { #[tokio::test] async fn an_unlicensed_read_records_its_bypass_reason() { - let (_node, sync) = solo("unlicensed-read"); + let (_node, sync) = solo("unlicensed-read").await; let proxy = proxy(Some(sync)); assert!(matches!( proxy.read_barrier(&HeaderMap::new()).await, @@ -492,7 +536,7 @@ async fn a_single_node_serves_a_suspect_copy_unproved() { /// atomic load, without the index being asked anything. #[tokio::test] async fn a_proved_copy_is_served_without_consulting_the_index() { - let (_node, sync) = solo("proved"); + let (_node, sync) = solo("proved").await; let proxy = proxy(Some(sync)); let obj = cached("v1", at(1_700_000_000)); obj.mark_trusted(proxy.obj_cache.suspect_gen()); @@ -579,7 +623,7 @@ async fn peer_put_cannot_complete_new_index_metadata_from_an_older_get() { /// and not once per read. The second call must not touch the index again. #[tokio::test] async fn a_suspect_copy_the_index_confirms_is_proved_exactly_once() { - let (_node, sync) = solo("confirm"); + let (_node, sync) = solo("confirm").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); proxy.obj_cache.insert(ck("k"), cached("v1", filled)).await; @@ -602,7 +646,7 @@ async fn a_suspect_copy_the_index_confirms_is_proved_exactly_once() { /// drop it and let the origin serve the read. #[tokio::test] async fn a_suspect_copy_the_index_contradicts_is_dropped() { - let (_node, sync) = solo("contradict"); + let (_node, sync) = solo("contradict").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); synced(&proxy); @@ -649,7 +693,7 @@ async fn a_suspect_copy_the_index_contradicts_is_dropped() { /// entry is stamped microseconds after its body. #[tokio::test] async fn a_byte_identical_rewrite_is_caught_by_the_mtime_and_a_fresh_fill_is_not() { - let (_node, sync) = solo("rewrite"); + let (_node, sync) = solo("rewrite").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); synced(&proxy); @@ -694,7 +738,7 @@ async fn a_byte_identical_rewrite_is_caught_by_the_mtime_and_a_fresh_fill_is_not /// are actually read. #[tokio::test] async fn an_unsynced_bucket_arbitrates_nothing_and_drops_the_copy() { - let (_node, sync) = solo("unsynced"); + let (_node, sync) = solo("unsynced").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); proxy.obj_cache.insert(ck("k"), cached("v1", filled)).await; diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index 4241c1b..d825667 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -1,15 +1,17 @@ +mod apply; + use std::num::NonZeroUsize; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime}; use groupnet::consistency::volatile_recovery::{RecoveryError, RecoveryHandle, RecoveryStatus}; use groupnet::consistency::{ - AckLedger, CAP_ACKS, CAP_LEASE, CoherenceOutcome, Frontier, LeaseConfig, LeaseView, Leases, - PeerWrite, PeerWrites, RenewalId, WriteFeed, WriteToken, advertised_head, applied_by_selected, + CAP_ACKS, CAP_LEASE, CoherenceOutcome, LeaseConfig, LeaseView, Leases, RenewalId, WriteFeed, + WriteToken, advertised_head, applied_by_selected, }; use groupnet::core::volatile_bootstrap::journal::NativeCut; use groupnet::core::{NodeId, Status}; -use groupnet::runtime::Group; +use groupnet::runtime::{Group, Node}; use s3s::dto::ObjectStorageClass; use tracing::{info, warn}; @@ -23,10 +25,8 @@ use crate::sync::readiness::UnmetSeeds; use crate::sync::stop::Crossing; use crate::sync::volatile::CacheRecoveryAdapter; use crate::sync::wire::{ - IndexEvent, IndexOp, decode_event, encode_event, etag_to_wire, from_micros, parse_token, - to_micros, + IndexEvent, IndexOp, encode_event, etag_to_wire, from_micros, parse_token, to_micros, }; -use crate::tier::LocalCache; /// Ring capacity: a peer that falls further behind than this many writes /// gets a gap (Groupnet closes serving, distrusts bodies, and rescans origin) instead @@ -163,7 +163,8 @@ impl Consistency { } /// Whether this mode participates in the ack tier — publishes an - /// [`AckLedger`] and waits on other participants' watermarks. Exhaustive + /// [`AckLedger`](groupnet::consistency::AckLedger) and waits on participants' + /// watermarks. Exhaustive /// on purpose: a mode added later has to answer this question rather than /// inherit an answer from a `!= Bounded`. pub(super) fn acks(self) -> bool { @@ -354,10 +355,9 @@ pub struct WriteSync { /// The deadline the coherence wait gets: one lease duration (the longest a /// silent holder's lapse can take) plus [`WRITE_WAIT_SLACK`]. write_wait: Duration, - /// Keeps the gossip node (receive loop, group actors) alive for the - /// process lifetime, whatever its transport. `None` when a test drives a raw - /// group directly. - _node: Option>, + /// Owns the managed gossip network for this feed. `None` when a test holds + /// the node separately while driving its group. + pub(super) node: Option, /// This node's `~caps` declaration; [`WriteSync::advertise_indexed`] extends it. pub(super) advertisement: Arc, /// Seeds not in the roster yet (see [`WriteSync::expect_seeds`]). @@ -389,7 +389,7 @@ impl WriteSync { me: NodeId, consistency: Consistency, lease: LeaseConfig, - node: Option>, + node: Option, ) -> Self { debug_assert!( lease.validate().is_ok(), @@ -418,7 +418,7 @@ impl WriteSync { advertisement, unmet_seeds: Arc::default(), absent_granters: std::sync::Mutex::default(), - _node: node, + node, } } @@ -898,136 +898,6 @@ impl WriteSync { .unwrap_or(false) } - /// Spawn the apply loop: peer events update the LIST index and invalidate - /// hot bodies. A peer write that describes nothing fences its key, and - /// `reconcile` resolves that fence from the origin. A gap synchronously - /// closes Groupnet's local serving gate, discards the index, and schedules - /// its rebuild. In `strong`, a small watcher reports the lease's monotone - /// lapse count to that same recovery driver. - /// - /// Takes `self` as an [`Arc`] because the lapse watcher holds a weak - /// reference back to this object, avoiding a task/handle cycle. - pub(crate) fn start_apply( - self: &Arc, - local: LocalCache, - state: Arc, - metrics: Arc, - reconcile: KeyReconcile, - ) { - let (frontier, view) = Frontier::new(); - let _ = self.view.set(view); - // Declare this node's feed writer before any capture can start, at - // its current position (zero for a feed with no writes yet). - state.register_native_writer(&self.own_position()); - if let Some(leases) = &self.leases { - let weak = Arc::downgrade(self); - let poll = lapse_poll(leases.config().duration); - tokio::spawn(async move { - loop { - tokio::time::sleep(poll).await; - let Some(sync) = weak.upgrade() else { return }; - let Some(recovery) = sync.recovery.get() else { - return; - }; - let count = sync.lease_view.as_ref().map_or(0, LeaseView::lapses); - if count > recovery.status().state.covered_lapses { - let _ = recovery.lease_lapse(count); - } - } - }); - } - // Bounded mode publishes no acks (that is its point at scale); both strong - // spellings do, and say so through their capability advertisement. - let ledger = self - .consistency - .acks() - .then(|| AckLedger::new(self.group.clone())); - let mut peers = PeerWrites::new(self.group.clone(), self.me.clone(), decode_event); - let weak = Arc::downgrade(self); - let lapse_view = self.lease_view.clone(); - tokio::spawn(async move { - while let Some(event) = peers.next().await { - let Some(sync) = weak.upgrade() else { return }; - match event { - PeerWrite::Sealed { peer, token } => { - state.note_native_seal(native_cut(&peer, token)); - sync.observe_seal(&peer, token); - frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } - } - PeerWrite::Renewed { - peer, - sealed, - epoch, - } => { - let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); - frontier.advance(&peer, start); - if let Some(ledger) = &ledger { - ledger.record(&peer, start).await; - } - } - PeerWrite::Wrote { - peer, - token, - key: event, - } => { - let cache_key = (event.bucket.clone(), event.key.clone()); - let mutation = local.fence_mutation(&cache_key).await; - let fenced = apply_feed_index(&state, &peer, token, &event); - drop(mutation); - // Fenced before the acknowledgement below: from then on this - // node answers the key from the origin until the reconciliation - // resolves it. - if let Some(fence) = fenced { - reconcile(&event.bucket, &event.key, fence); - } - // The index must move first: a warm body promoted after this hot - // eviction decodes suspect and is checked against the new entry - // before it can be served. Awaiting moka removal before the ack - // closes the stale-hot window without putting disk I/O on the - // feed frontier. - local.invalidate_hot(&cache_key).await; - frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } - metrics.feed_applied(); - // The index has moved past any copy this node holds, so the warm - // copy can never serve again; left in place it would only crowd - // live bodies out of the disk budget until LRU found it. Disk I/O - // stays off the frontier: retire it after the acknowledgement. - let retire = local.clone(); - tokio::spawn(async move { retire.invalidate(&cache_key).await }); - } - PeerWrite::Gap { - peer, - missed_through, - } => { - warn!("write-feed gap from `{peer}`: distrusting bodies, resyncing index"); - sync.forget_crossing(&peer); - state.discard_for_gap(&native_cut(&peer, missed_through)); - // The public signal fences serving before returning; the - // recovery worker distrusts bodies and guards every origin - // scan page with that exact operation's permit. Frontier - // progress below means the old local state is unservable, - // not that the replacement index has materialized already. - if let Some(recovery) = sync.recovery.get() { - let lapses = lapse_view.as_ref().map_or(0, LeaseView::lapses); - let _ = recovery.feed_gap(lapses); - } - frontier.advance(&peer, missed_through); - if let Some(ledger) = &ledger { - ledger.record(&peer, missed_through).await; - } - metrics.feed_gap(); - } - } - } - }); - } - /// Wait (bounded by `timeout`) until every peer's currently-advertised /// feed head has been applied locally. Returns whether it fully caught /// up; freshness is bounded by one push/gossip hop (see module docs). diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs new file mode 100644 index 0000000..42790f5 --- /dev/null +++ b/src/sync/coherence/apply.rs @@ -0,0 +1,142 @@ +use std::sync::Arc; + +use groupnet::consistency::{AckLedger, Frontier, LeaseView, PeerWrite, PeerWrites}; +use tracing::warn; + +use crate::index::KeyIndex; +use crate::metrics::Metrics; +use crate::sync::coherence::{KeyReconcile, WriteSync, apply_feed_index, lapse_poll, native_cut}; +use crate::sync::wire::decode_event; +use crate::tier::LocalCache; + +impl WriteSync { + /// Spawn the apply loop: peer events update the LIST index and invalidate + /// hot bodies. A peer write that describes nothing fences its key, and + /// `reconcile` resolves that fence from the origin. A gap synchronously + /// closes Groupnet's local serving gate, discards the index, and schedules + /// its rebuild. In `strong`, a small watcher reports the lease's monotone + /// lapse count to that same recovery driver. + /// + /// Takes `self` as an [`Arc`] because the lapse watcher holds a weak + /// reference back to this object, avoiding a task/handle cycle. + pub(crate) fn start_apply( + self: &Arc, + local: LocalCache, + state: Arc, + metrics: Arc, + reconcile: KeyReconcile, + ) { + let (frontier, view) = Frontier::new(); + let _ = self.view.set(view); + // Declare this node's feed writer before any capture can start, at + // its current position (zero for a feed with no writes yet). + state.register_native_writer(&self.own_position()); + if let Some(leases) = &self.leases { + let weak = Arc::downgrade(self); + let poll = lapse_poll(leases.config().duration); + tokio::spawn(async move { + loop { + tokio::time::sleep(poll).await; + let Some(sync) = weak.upgrade() else { return }; + let Some(recovery) = sync.recovery.get() else { + return; + }; + let count = sync.lease_view.as_ref().map_or(0, LeaseView::lapses); + if count > recovery.status().state.covered_lapses { + let _ = recovery.lease_lapse(count); + } + } + }); + } + // Bounded mode publishes no acks (that is its point at scale); both strong + // spellings do, and say so through their capability advertisement. + let ledger = self + .consistency + .acks() + .then(|| AckLedger::new(self.group.clone())); + let mut peers = PeerWrites::new(self.group.clone(), self.me.clone(), decode_event); + let weak = Arc::downgrade(self); + let lapse_view = self.lease_view.clone(); + tokio::spawn(async move { + while let Some(event) = peers.next().await { + let Some(sync) = weak.upgrade() else { return }; + match event { + PeerWrite::Sealed { peer, token } => { + state.note_native_seal(native_cut(&peer, token)); + sync.observe_seal(&peer, token); + frontier.advance(&peer, token); + if let Some(ledger) = &ledger { + ledger.record(&peer, token).await; + } + } + PeerWrite::Renewed { + peer, + sealed, + epoch, + } => { + let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); + frontier.advance(&peer, start); + if let Some(ledger) = &ledger { + ledger.record(&peer, start).await; + } + } + PeerWrite::Wrote { + peer, + token, + key: event, + } => { + let cache_key = (event.bucket.clone(), event.key.clone()); + let mutation = local.fence_mutation(&cache_key).await; + let fenced = apply_feed_index(&state, &peer, token, &event); + drop(mutation); + // Fenced before the acknowledgement below: from then on this + // node answers the key from the origin until the reconciliation + // resolves it. + if let Some(fence) = fenced { + reconcile(&event.bucket, &event.key, fence); + } + // The index must move first: a warm body promoted after this hot + // eviction decodes suspect and is checked against the new entry + // before it can be served. Awaiting moka removal before the ack + // closes the stale-hot window without putting disk I/O on the + // feed frontier. + local.invalidate_hot(&cache_key).await; + frontier.advance(&peer, token); + if let Some(ledger) = &ledger { + ledger.record(&peer, token).await; + } + metrics.feed_applied(); + // The index has moved past any copy this node holds, so the warm + // copy can never serve again; left in place it would only crowd + // live bodies out of the disk budget until LRU found it. Disk I/O + // stays off the frontier: retire it after the acknowledgement. + let retire = local.clone(); + tokio::spawn(async move { retire.invalidate(&cache_key).await }); + } + PeerWrite::Gap { + peer, + missed_through, + } => { + warn!("write-feed gap from `{peer}`: distrusting bodies, resyncing index"); + sync.forget_crossing(&peer); + state.discard_for_gap(&native_cut(&peer, missed_through)); + // The public signal fences serving before returning; the + // recovery worker distrusts bodies and guards every origin + // scan page with that exact operation's permit. Frontier + // progress below means the old local state is unservable, + // not that the replacement index has materialized already. + if let Some(recovery) = sync.recovery.get() { + let lapses = lapse_view.as_ref().map_or(0, LeaseView::lapses); + let _ = recovery.feed_gap(lapses); + } + frontier.advance(&peer, missed_through); + if let Some(ledger) = &ledger { + ledger.record(&peer, missed_through).await; + } + metrics.feed_gap(); + } + } + } + }); + } +} diff --git a/src/sync/config.rs b/src/sync/config.rs index 8122220..96ece33 100644 --- a/src/sync/config.rs +++ b/src/sync/config.rs @@ -4,7 +4,7 @@ use std::time::Duration; use groupnet::consistency::LeaseConfig; use groupnet::core::{Config, NodeId}; use groupnet::runtime::{Group, NamedSeeds, Node, SeedEvent, SystemResolver}; -use groupnet::transport::Transport; +use groupnet::transport::link::{BoundLink, LinkConfig, LinkProvider}; use groupnet::transport::udp::UdpTransport; use tracing::{info, warn}; @@ -63,9 +63,9 @@ pub struct SyncConfig { /// The address peers should reach this node on; the bound address when /// `None` (`S3CACHE_GOSSIP_ADVERTISE`). pub advertise: Option, - /// Statically-addressed peers as `(node id, host:port)`. Every other peer - /// resolves itself through gossiped advertisements, so only seeds need - /// static addressing (`S3CACHE_GOSSIP_SEEDS`). + /// Explicit routing neighbors as `(node id, host:port)`. DNS resolution and + /// gossiped advertisements refresh their addresses, but do not admit additional + /// neighbors (`S3CACHE_GOSSIP_SEEDS`). pub seeds: Vec<(String, String)>, /// This node's identity in the cluster (the pod name, in the chart). pub node_id: String, @@ -145,60 +145,62 @@ impl WriteSync { for (id, addr) in &cfg.seeds { resolution = resolution.seed(NodeId::new(id.as_str()), addr.as_str()); } - let mut builder = Node::builder(me.clone(), transport) + let mut builder = Node::builder(me.clone()) + .link(BoundLink::new(transport, LinkConfig::new(seeds.clone()))) .config(gossip_config(cfg.lease_ms)) .named_seeds(resolution); if let Some(advertise) = advertise { builder = builder.advertise_addr(advertise); } - let node = builder.spawn(); + let node = match builder.start().await { + Ok(node) => node, + Err(error) => { + warn!("gossip disabled: cannot start managed network: {error}"); + return None; + } + }; let group = node.join_group("s3cache"); let (bind, node_id, mode) = (&cfg.bind, &cfg.node_id, cfg.consistency.label()); let lease_ms = cfg.lease_ms; info!( "gossip coherence bound on `{bind}` as `{node_id}` (consistency: {mode}, lease: {lease_ms}ms)" ); - let sync = WriteSync::attach(group, me, cfg.consistency, lease, Some(Box::new(node))); + let sync = WriteSync::attach(group, me, cfg.consistency, lease, Some(node)); let _ = unmet.set(sync.expect_seeds(seeds)); Some(sync) } - /// Join the cluster group as `node_id` over an already-built groupnet - /// `transport`, seeded with `seeds`, with exactly the membership tuning and - /// coherence lease [`WriteSync::new`] runs. `new` is this over UDP plus seed - /// address resolution; any other transport (the in-memory one, for a whole - /// fleet in one process) resolves its own peers. + /// Join the cluster group as `node_id` over a managed protocol `link`, with + /// exactly the membership tuning and coherence lease [`WriteSync::new`] runs. + /// The link explicitly admits its routing neighbors; `seeds` additionally + /// records the peers readiness must await. /// - /// # Panics - /// Outside a Tokio runtime: the node spawns its tasks here. - pub fn over_transport( - transport: T, + /// # Errors + /// Returns an error if the link cannot bind or the managed network cannot start. + pub async fn over_link( + link: impl LinkProvider, node_id: &str, seeds: &[&str], consistency: Consistency, lease_ms: u64, - ) -> Self { + ) -> std::io::Result { let me = NodeId::new(node_id); - let mut builder = Node::builder(me.clone(), transport).config(gossip_config(lease_ms)); + let mut builder = Node::builder(me.clone()) + .link(link) + .config(gossip_config(lease_ms)); for seed in seeds.iter().filter(|seed| **seed != node_id) { builder = builder.seed(NodeId::new(*seed)); } - let node = builder.spawn(); + let node = builder.start().await?; let group = node.join_group("s3cache"); - let sync = WriteSync::attach( - group, - me, - consistency, - lease_config(lease_ms), - Some(Box::new(node)), - ); + let sync = WriteSync::attach(group, me, consistency, lease_config(lease_ms), Some(node)); sync.expect_seeds( seeds .iter() .filter(|seed| **seed != node_id) .map(|seed| NodeId::new(*seed)), ); - sync + Ok(sync) } /// The gossip group this node's feed, leases and membership ride on, for diff --git a/src/sync/stop.rs b/src/sync/stop.rs index f14eb5a..d90fce3 100644 --- a/src/sync/stop.rs +++ b/src/sync/stop.rs @@ -85,6 +85,14 @@ impl WriteSync { } } + /// Close the managed gossip network after the mutation drain and feed seal. + /// This cancels address resolution and releases every owned link endpoint. + pub async fn close(&self) { + if let Some(node) = &self.node { + node.close().await; + } + } + /// Advertise this life's feed epoch before any write, so every peer learns of the /// restart now: a sealed previous life renews, an unsealed one gaps at once. pub(crate) async fn announce(&self) { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 73001ec..2ac1005 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -5,8 +5,8 @@ use groupnet::consistency::volatile_recovery::{RecoveryStage, RecoveryStatus}; use groupnet::consistency::{CAP_ACKS, CAP_LEASE, LeaseConfig}; use groupnet::core::volatile_recovery::RecoveryState; use groupnet::core::{Config, NodeId, Status}; -use groupnet::runtime::{Group, NamedSeeds, Node, ResolveFuture, SeedResolver, SystemResolver}; -use groupnet::transport::mem::{MemTransport, Network}; +use groupnet::runtime::{NamedSeeds, ResolveFuture, SeedResolver, SystemResolver}; +use groupnet::transport::mem::Network; use s3s::dto::GetObjectOutput; use crate::index::{KeyIndex, ObjEntry, standard_class}; @@ -21,6 +21,9 @@ use crate::sync::wire::{ }; use crate::tier::{CachedObject, TieredCache, open_warm}; +mod fixtures; +use fixtures::{attach, spawn_neighbors, spawn_node, wired_pair, wired_pair_named}; + type Index = Arc; /// A lease short enough to watch lapse inside a test, and still comfortably inside @@ -112,22 +115,6 @@ fn only_acknowledged_strong_waits_retire_feed_history() { assert!(!WriteWait::Lapsed(Vec::new()).retires_feed(Consistency::Bounded)); } -fn spawn_node(net: &Network, id: &str, peer: &str) -> (NodeId, Node, Group) { - let me = NodeId::new(id); - let node = Node::builder(me.clone(), net.endpoint(me.clone())) - .seed(NodeId::new(peer)) - .config(brisk()) - .spawn(); - let group = node.join_group("s3cache"); - (me, node, group) -} - -/// A [`WriteSync`] on `group`, leased at [`TEST_LEASE`] unless the mode says -/// otherwise. -fn attach(group: Group, me: NodeId, consistency: Consistency) -> WriteSync { - WriteSync::attach(group, me, consistency, test_lease(), None) -} - fn cached(body: &'static [u8]) -> Arc { Arc::new(CachedObject::from_get( &GetObjectOutput::default(), @@ -154,33 +141,6 @@ fn indexed(state: &Index, bucket: &str, key: &str) -> Option { .cloned() } -/// A fully-wired pair: node A publishes, node B applies into `state` and -/// its local cache. Returns A's publisher, B's sync, and B's state and cache. -fn wired_pair(net: &Network) -> (WriteSync, Arc, Index, TieredCache) { - wired_pair_named(net, ("sync-a", "sync-b"), Consistency::Strong) -} - -/// [`wired_pair`] with the gossip identities and the mode spelled out — unique ids -/// per test, so parallel tests never share one. -/// -/// B comes back as an [`Arc`]: it is the node whose apply loop runs, and -/// [`WriteSync::start_apply`] hands the lapse watch a [`Weak`] back to it. -fn wired_pair_named( - net: &Network, - ids: (&str, &str), - consistency: Consistency, -) -> (WriteSync, Arc, Index, TieredCache) { - let (a_id, _a_node, a_group) = spawn_node(net, ids.0, ids.1); - let (b_id, _b_node, b_group) = spawn_node(net, ids.1, ids.0); - let metrics = Arc::new(Metrics::default()); - let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); - let state = Arc::new(KeyIndex::default()); - let sync_b = Arc::new(attach(b_group, b_id, consistency)); - sync_b.start_apply(cache.local(), state.clone(), metrics, no_reconcile()); - let sync_a = attach(a_group, a_id, consistency); - (sync_a, sync_b, state, cache) -} - async fn eventually(mut cond: impl FnMut() -> bool, what: &str) { for _ in 0..300 { if cond() { @@ -321,8 +281,8 @@ async fn an_unresolvable_seed_releases_readiness_after_its_window() { } /// A seed that comes back at a new address is relearned by re-resolution, and -/// this node reaches the new life — which, seeding nobody, would otherwise -/// never hear from it. +/// this node reaches the new life. Both endpoints explicitly admit each other; +/// discovering an address does not admit a new routing neighbor. #[tokio::test(flavor = "multi_thread")] async fn a_moved_seed_is_relearned_and_rejoins() { let (a, b_first, b_moved) = (loopback(), loopback(), loopback()); @@ -334,9 +294,10 @@ async fn a_moved_seed_is_relearned_and_rejoins() { let node_a = WriteSync::bind(seed_config("seed-a", a, &[("seed-b", "b:1")]), resolution) .await .expect("binds loopback"); + let address_a = a.to_string(); let peer = |bind| { WriteSync::bind( - seed_config("seed-b", bind, &[]), + seed_config("seed-b", bind, &[("seed-a", address_a.as_str())]), NamedSeeds::new(SystemResolver), ) }; @@ -393,11 +354,12 @@ fn only_a_mode_that_grants_leases_advertises_that_it_does() { #[tokio::test] async fn the_ack_wait_covers_advertisers_and_unknowns_but_not_declared_bounded_peers() { let net = Network::new(); - let (_a_id, _a_node, a_group) = spawn_node(&net, "cap-a", "cap-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "cap-b", "cap-a"); - let (c_id, _c_node, c_group) = spawn_node(&net, "cap-c", "cap-a"); + let (_a_id, _a_node, a_group) = + spawn_neighbors(&net, "cap-a", &["cap-b", "cap-c", "cap-d"]).await; + let (b_id, _b_node, b_group) = spawn_node(&net, "cap-b", "cap-a").await; + let (c_id, _c_node, c_group) = spawn_node(&net, "cap-c", "cap-a").await; // D runs, joins, and never advertises — a node from before this change. - let (d_id, _d_node, _d_group) = spawn_node(&net, "cap-d", "cap-a"); + let (d_id, _d_node, _d_group) = spawn_node(&net, "cap-d", "cap-a").await; b_group .advertise_capabilities([CAP_ACKS]) .expect("B advertises the ack tier"); @@ -427,12 +389,13 @@ async fn the_ack_wait_covers_advertisers_and_unknowns_but_not_declared_bounded_p #[tokio::test] async fn the_transition_wait_covers_exactly_what_the_lease_cannot() { let net = Network::new(); - let (_a_id, _a_node, a_group) = spawn_node(&net, "mix-a", "mix-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "mix-b", "mix-a"); - let (c_id, _c_node, c_group) = spawn_node(&net, "mix-c", "mix-a"); + let (_a_id, _a_node, a_group) = + spawn_neighbors(&net, "mix-a", &["mix-b", "mix-c", "mix-d", "mix-e"]).await; + let (b_id, _b_node, b_group) = spawn_node(&net, "mix-b", "mix-a").await; + let (c_id, _c_node, c_group) = spawn_node(&net, "mix-c", "mix-a").await; // D runs, joins, and never advertises — a node from before any of this. - let (d_id, _d_node, _d_group) = spawn_node(&net, "mix-d", "mix-a"); - let (e_id, _e_node, e_group) = spawn_node(&net, "mix-e", "mix-a"); + let (d_id, _d_node, _d_group) = spawn_node(&net, "mix-d", "mix-a").await; + let (e_id, _e_node, e_group) = spawn_node(&net, "mix-e", "mix-a").await; b_group .advertise_capabilities(Consistency::StrongAcks.capabilities().iter().copied()) .expect("B pins the ack tier"); @@ -528,7 +491,7 @@ fn local_stamps_are_truncated_to_the_wire_precision() { #[tokio::test] async fn peer_events_fold_into_index_and_invalidate() { let net = Network::new(); - let (sync_a, _sync_b, state, cache) = wired_pair(&net); + let (sync_a, _sync_b, state, cache) = wired_pair(&net).await; let metrics = Metrics::default(); // B holds a soon-stale body copy. @@ -588,8 +551,8 @@ async fn peer_events_fold_into_index_and_invalidate() { #[tokio::test] async fn peer_delete_retires_the_warm_copy() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "warm-a", "warm-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "warm-b", "warm-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "warm-a", "warm-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "warm-b", "warm-a").await; let metrics = Arc::new(Metrics::default()); let dir = std::env::temp_dir().join(format!("s3cache-peer-warm-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); @@ -646,8 +609,8 @@ async fn peer_delete_retires_the_warm_copy() { #[tokio::test] async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "unknown-a", "unknown-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "unknown-b", "unknown-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "unknown-a", "unknown-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "unknown-b", "unknown-a").await; let metrics = Arc::new(Metrics::default()); let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); let state = Arc::new(KeyIndex::default()); @@ -713,7 +676,7 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { #[tokio::test] async fn await_fresh_reflects_the_publishers_head() { let net = Network::new(); - let (sync_a, sync_b, state, _cache) = wired_pair(&net); + let (sync_a, sync_b, state, _cache) = wired_pair(&net).await; let metrics = Metrics::default(); own_put(&sync_a, "fresh", written(7), &metrics).await; @@ -737,7 +700,7 @@ async fn await_fresh_reflects_the_publishers_head() { #[tokio::test] async fn write_tokens_upgrade_reads_to_strict() { let net = Network::new(); - let (sync_a, sync_b, state, _cache) = wired_pair(&net); + let (sync_a, sync_b, state, _cache) = wired_pair(&net).await; let metrics = Metrics::default(); let receipt = own_put(&sync_a, "tok", written(1), &metrics).await; @@ -793,7 +756,7 @@ async fn write_tokens_upgrade_reads_to_strict() { async fn in_strong_acks_an_unacked_write_is_counted_and_not_retired() { let net = Network::new(); let (_sync_a, sync_b, _state, _cache) = - wired_pair_named(&net, ("acks-a", "acks-b"), Consistency::StrongAcks); + wired_pair_named(&net, ("acks-a", "acks-b"), Consistency::StrongAcks).await; let metrics = Metrics::default(); // A was attached without an apply loop, so it publishes no ack ledger — but it // is alive, so B has to wait for it and then give up. @@ -843,7 +806,7 @@ async fn in_strong_acks_an_unacked_write_is_counted_and_not_retired() { async fn a_leased_write_resolves_on_acks_and_the_peer_already_has_it() { let net = Network::new(); let (sync_a, _sync_b, state, _cache) = - wired_pair_named(&net, ("fast-a", "fast-b"), Consistency::Strong); + wired_pair_named(&net, ("fast-a", "fast-b"), Consistency::Strong).await; let metrics = Metrics::default(); // Nothing about a *fast* path is provable until there is somebody to be fast // against: an empty wait set resolves immediately and would pass this vacuously. @@ -883,8 +846,8 @@ async fn a_leased_write_resolves_on_acks_and_the_peer_already_has_it() { #[tokio::test] async fn a_dropped_peers_lease_lapses_and_the_write_completes_inside_one_duration() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "lapse-a", "lapse-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "lapse-b", "lapse-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "lapse-a", "lapse-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "lapse-b", "lapse-a").await; let sync_a = attach(a_group, a_id, Consistency::Strong); // B renews and grants but never applies: no apply loop, so no ack ledger. It is // the fail-slow reader, which is exactly the peer a lapse has to rescue. @@ -954,8 +917,8 @@ async fn a_dropped_peers_lease_lapses_and_the_write_completes_inside_one_duratio #[tokio::test] async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "absent-a", "absent-b"); - let (b_id, b_node, b_group) = spawn_node(&net, "absent-b", "absent-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "absent-a", "absent-b").await; + let (b_id, b_node, b_group) = spawn_node(&net, "absent-b", "absent-a").await; let sync_a = attach(a_group, a_id, Consistency::Strong); let sync_b = attach(b_group, b_id.clone(), Consistency::Strong); eventually( @@ -968,12 +931,10 @@ async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { "B is alive and granting" ); - // A crash: no departure, the node simply stops. Dropping the handles is not enough - // on its own — the receive loop keeps the node ticking until its endpoint is - // evicted, which registering the id again does. + // A crash: no departure or feed seal; close only the managed network. drop(sync_b); + b_node.close().await; drop(b_node); - drop(net.endpoint(b_id.clone())); eventually( || sync_a.absent_granters() == [b_id.clone()], "A to name B as the counted granter it is waiting on", @@ -985,7 +946,7 @@ async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { assert!(!sync_a.may_serve_local()); // B returns under the same id (a StatefulSet pod) and is no longer named. - let (b_id, _b_node, b_group) = spawn_node(&net, "absent-b", "absent-a"); + let (b_id, _b_node, b_group) = spawn_node(&net, "absent-b", "absent-a").await; let _sync_b = attach(b_group, b_id, Consistency::Strong); eventually( || sync_a.absent_granters().is_empty(), diff --git a/src/sync/tests/fixtures.rs b/src/sync/tests/fixtures.rs new file mode 100644 index 0000000..70bce58 --- /dev/null +++ b/src/sync/tests/fixtures.rs @@ -0,0 +1,76 @@ +use std::sync::Arc; +use std::time::Duration; + +use groupnet::core::NodeId; +use groupnet::network::RouterConfig; +use groupnet::runtime::{Group, Node}; +use groupnet::transport::link::{BoundLink, LinkConfig}; +use groupnet::transport::mem::Network; + +use crate::index::KeyIndex; +use crate::metrics::Metrics; +use crate::sync::coherence::{Consistency, WriteSync}; +use crate::tier::TieredCache; + +use super::{Index, brisk, no_reconcile, test_lease}; + +pub(super) async fn spawn_node(net: &Network, id: &str, peer: &str) -> (NodeId, Node, Group) { + spawn_neighbors(net, id, &[peer]).await +} + +pub(super) async fn spawn_neighbors( + net: &Network, + id: &str, + peers: &[&str], +) -> (NodeId, Node, Group) { + let me = NodeId::new(id); + let peers = peers.iter().map(|peer| NodeId::new(*peer)).collect(); + let node = Node::builder(me.clone()) + .link(BoundLink::new( + net.endpoint(me.clone()), + LinkConfig::new(peers), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) + .config(brisk()) + .start() + .await + .expect("managed memory node starts"); + let group = node.join_group("s3cache"); + (me, node, group) +} + +/// A feed whose caller retains the managed node separately. +pub(super) fn attach(group: Group, me: NodeId, consistency: Consistency) -> WriteSync { + WriteSync::attach(group, me, consistency, test_lease(), None) +} + +/// A fully-wired pair: A publishes, B applies into its local state and cache. +pub(super) async fn wired_pair(net: &Network) -> (WriteSync, Arc, Index, TieredCache) { + wired_pair_named(net, ("sync-a", "sync-b"), Consistency::Strong).await +} + +/// Each returned feed owns its node, including the managed network lifetime. +pub(super) async fn wired_pair_named( + net: &Network, + ids: (&str, &str), + consistency: Consistency, +) -> (WriteSync, Arc, Index, TieredCache) { + let (a_id, a_node, a_group) = spawn_node(net, ids.0, ids.1).await; + let (b_id, b_node, b_group) = spawn_node(net, ids.1, ids.0).await; + let metrics = Arc::new(Metrics::default()); + let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); + let state = Arc::new(KeyIndex::default()); + let sync_b = Arc::new(WriteSync::attach( + b_group, + b_id, + consistency, + test_lease(), + Some(b_node), + )); + sync_b.start_apply(cache.local(), state.clone(), metrics, no_reconcile()); + let sync_a = WriteSync::attach(a_group, a_id, consistency, test_lease(), Some(a_node)); + (sync_a, sync_b, state, cache) +} diff --git a/src/sync/volatile.rs b/src/sync/volatile.rs index 7b60634..d8b9a6f 100644 --- a/src/sync/volatile.rs +++ b/src/sync/volatile.rs @@ -356,7 +356,9 @@ mod tests { use super::*; use groupnet::consistency::LeaseConfig; use groupnet::core::Config; + use groupnet::network::RouterConfig; use groupnet::runtime::Node; + use groupnet::transport::link::{BoundLink, LinkConfig}; use groupnet::transport::mem::Network; #[tokio::test] @@ -369,14 +371,30 @@ mod tests { anti_entropy_interval_ms: 25, ..Config::default() }; - let node_a = Node::builder(a.clone(), net.endpoint(a.clone())) - .seed(b.clone()) + let routing = RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }; + let node_a = Node::builder(a.clone()) + .link(BoundLink::new( + net.endpoint(a.clone()), + LinkConfig::new(vec![b.clone()]), + )) + .routing(routing.clone()) .config(config.clone()) - .spawn(); - let node_b = Node::builder(b.clone(), net.endpoint(b.clone())) - .seed(a.clone()) + .start() + .await + .expect("A managed node starts"); + let node_b = Node::builder(b.clone()) + .link(BoundLink::new( + net.endpoint(b.clone()), + LinkConfig::new(vec![a.clone()]), + )) + .routing(routing) .config(config) - .spawn(); + .start() + .await + .expect("B managed node starts"); let group_a = node_a.join_group("s3cache"); let group_b = node_b.join_group("s3cache"); tokio::time::timeout(Duration::from_secs(3), async { diff --git a/tests/it/common/fleet.rs b/tests/it/common/fleet.rs index 79f1231..619d5d0 100644 --- a/tests/it/common/fleet.rs +++ b/tests/it/common/fleet.rs @@ -252,13 +252,22 @@ pub async fn mem_pod_node( let (name, peer) = (name.to_owned(), peer.to_owned()); pod.handle() .spawn(async move { - let sync = Arc::new(WriteSync::over_transport( - net.endpoint(groupnet::core::NodeId::new(name.as_str())), - &name, - &[peer.as_str()], - Consistency::Strong, - DEFAULT_LEASE_MS, - )); + let sync = Arc::new( + WriteSync::over_link( + groupnet::transport::link::BoundLink::new( + net.endpoint(groupnet::core::NodeId::new(name.as_str())), + groupnet::transport::link::LinkConfig::new(vec![ + groupnet::core::NodeId::new(peer.as_str()), + ]), + ), + &name, + &[peer.as_str()], + Consistency::Strong, + DEFAULT_LEASE_MS, + ) + .await + .expect("managed memory link starts"), + ); let proxy = proxy_over_with_metrics( &origin.counted_client(), CAP, diff --git a/tests/it/strong_rw.rs b/tests/it/strong_rw.rs index d5326ba..536182c 100644 --- a/tests/it/strong_rw.rs +++ b/tests/it/strong_rw.rs @@ -23,24 +23,24 @@ use crate::common; +mod workload; +use workload::{delete, get, head, list, put}; + use std::collections::BTreeMap; use std::fmt::Write as _; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; use bytes::Bytes; -use common::faultnet::{FaultNet, Link}; +use common::faultnet::FaultNet; use common::fleet::{CAP, Pod, SERIAL_SCAN, fleet_config_bound, free_tcp_port, serves_locally}; -use common::{Origin, WarmDir, body_blob, request}; +use common::{Origin, WarmDir}; use s3cache::cache::proxy::{CacheConfig, CachingProxy}; use s3cache::metrics::Metrics; use s3cache::sync::coherence::{Consistency, DEFAULT_LEASE_MS, WriteSync}; -use s3cache::tier::{buffer_body, open_warm}; -use s3s::dto::{ - DeleteObjectInput, ETag, ETagCondition, GetObjectInput, HeadObjectInput, ListObjectsV2Input, - PutObjectInput, Range, -}; -use s3s::{S3, S3Error, S3ErrorCode}; +use s3cache::tier::open_warm; +use s3s::S3ErrorCode; +use s3s::dto::{ETag, ETagCondition}; /// A hot tier smaller than the workload's bodies together, so most reads find /// their copy on the warm tier and have to prove it. @@ -240,13 +240,22 @@ impl Cluster { let proxy = pod .handle() .spawn(async move { - let sync = Arc::new(WriteSync::over_transport( - transport, - name, - &[peer], - Consistency::Strong, - DEFAULT_LEASE_MS, - )); + let sync = Arc::new( + WriteSync::over_link( + groupnet::transport::link::BoundLink::new( + transport, + groupnet::transport::link::LinkConfig::new(vec![ + groupnet::core::NodeId::new(peer), + ]), + ), + name, + &[peer], + Consistency::Strong, + DEFAULT_LEASE_MS, + ) + .await + .expect("managed fault link starts"), + ); let proxy = CachingProxy::new( &client, CacheConfig { @@ -907,642 +916,3 @@ impl Cond { } } } - -async fn put( - proxy: &CachingProxy, - bucket: &str, - key: &str, - payload: Bytes, - if_match: Option, - if_none_match: Option, -) -> Result, S3Error> { - let version = version_of(&payload).unwrap_or_default(); - // Every fourth version names no Content-Type: the proxy indexes it skeletally - // and keeps no copy, so its first HEAD and GET are the origin's to answer. - let content_type = - (!version.is_multiple_of(4)).then(|| "application/x-s3cache-check".to_owned()); - let input = PutObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - content_length: Some(i64::try_from(payload.len()).expect("small")), - body: Some(body_blob(payload)), - content_type, - metadata: Some([("version".to_owned(), version.to_string())].into()), - if_match, - if_none_match, - ..Default::default() - }; - let out = proxy.put_object(request(input)).await?.output; - Ok(out.e_tag.map(ETag::into_value)) -} - -async fn delete(proxy: &CachingProxy, bucket: &str, key: &str) -> Result<(), S3Error> { - let input = DeleteObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - ..Default::default() - }; - proxy.delete_object(request(input)).await?; - Ok(()) -} - -async fn get( - proxy: &CachingProxy, - bucket: &str, - key: &str, - range: Option<(u64, u64)>, -) -> Result { - let input = GetObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - range: range.map(|(first, last)| Range::Int { - first, - last: Some(last), - }), - ..Default::default() - }; - match proxy.get_object(request(input)).await { - Ok(mut out) => { - let blob = out.output.body.take().expect("a GET carries a body"); - let bytes = buffer_body(blob, usize::MAX).await.expect("readable body"); - Ok(version_of(&bytes).map_or(Value::Absent, Value::V)) - } - Err(error) if error.code() == &S3ErrorCode::NoSuchKey => Ok(Value::Absent), - Err(error) => Err(error), - } -} - -async fn head(proxy: &CachingProxy, bucket: &str, key: &str) -> Result { - let input = HeadObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - ..Default::default() - }; - match proxy.head_object(request(input)).await { - Ok(out) => { - let out = out.output; - Ok(Seen::Head { - etag: out.e_tag.map(ETag::into_value), - meta: out - .metadata - .as_ref() - .and_then(|meta| meta.get("version")) - .and_then(|version| version.parse().ok()), - length: out.content_length, - }) - } - Err(error) - if error.code() == &S3ErrorCode::NoSuchKey - || error.status_code() == Some(http::StatusCode::NOT_FOUND) => - { - Ok(Seen::Value(Value::Absent)) - } - Err(error) => Err(error), - } -} - -type Rows = BTreeMap, Option)>; - -async fn list(proxy: &CachingProxy, bucket: &str, prefix: &str) -> Result { - let out = proxy - .list_objects_v2(request(ListObjectsV2Input { - bucket: bucket.to_owned(), - prefix: Some(prefix.to_owned()), - ..Default::default() - })) - .await? - .output; - Ok(out - .contents - .into_iter() - .flatten() - .filter_map(|object| { - Some(( - object.key?, - (object.e_tag.map(ETag::into_value), object.size), - )) - }) - .collect()) -} - -/// A seeded splitmix64 stream per workload task. -struct Rng(u64); - -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - fn below(&mut self, bound: u64) -> u64 { - self.next() % bound - } - - fn node(&mut self) -> usize { - usize::from(self.next() % 2 == 1) - } -} - -/// The other node, unless it is down: then the same one. -fn other(cluster: &Cluster, via: usize) -> usize { - if cluster.node(1 - via).is_some() { - 1 - via - } else { - via - } -} - -/// A node that is up, preferring `via`. -fn up(cluster: &Cluster, via: usize) -> usize { - if cluster.node(via).is_some() { - via - } else { - 1 - via - } -} - -fn read_kind(rng: &mut Rng) -> ReadKind { - match rng.below(3) { - 0 => ReadKind::Get, - 1 => ReadKind::Range, - _ => ReadKind::Head, - } -} - -/// Overwrites of the pointer keys through alternating nodes, each followed by a -/// read through the other node. -async fn pointer_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut version = 0; - while Instant::now() < until { - version += 1; - let key = POINTERS[usize::try_from(version % 2).expect("small")]; - let via = up(&cluster, rng.node()); - if cluster.write(via, key, Value::V(version), Cond::None).await == WriteStatus::Acked { - let reader = other(&cluster, via); - cluster.read(reader, key, read_kind(&mut rng)).await; - if rng.below(4) == 0 { - cluster.list(reader, "ptr/").await; - } - } - } -} - -/// An `If-Match` chain: HEAD through one node, conditional overwrite through the -/// other, the way a log writer advances its pointer. -async fn cas_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut version = 1_000_000; - while Instant::now() < until { - let reader = up(&cluster, rng.node()); - let (bucket, key) = (cluster.bucket.clone(), CAS.to_owned()); - let invoked = Instant::now(); - let answer = cluster - .on(reader, move |proxy| async move { - head(&proxy, &bucket, &key).await - }) - .await - .and_then(Result::ok); - let done = Instant::now(); - let Some(current) = answer else { continue }; - cluster.history.lock().unwrap().reads.push(Read { - key: CAS.to_owned(), - op: "HEAD (CAS)", - via: reader, - invoked, - done, - seen: current.clone(), - }); - version += 1; - let cond = match current { - Seen::Head { - etag: Some(etag), .. - } => Cond::IfMatch(etag), - _ => Cond::IfNoneMatch, - }; - let writer = other(&cluster, reader); - cluster.write(writer, CAS, Value::V(version), cond).await; - } -} - -/// Log slots: create once (a contested duplicate through the other node must -/// lose), read and list through the other node, truncate old slots behind. -async fn slot_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut slot = 0_u64; - let mut version = 2_000_000; - while Instant::now() < until { - slot += 1; - let key = format!("log/{slot:04}"); - let via = up(&cluster, rng.node()); - version += 1; - if cluster - .write(via, &key, Value::V(version), Cond::IfNoneMatch) - .await - != WriteStatus::Acked - { - continue; - } - let reader = other(&cluster, via); - cluster.read(reader, &key, read_kind(&mut rng)).await; - version += 1; - cluster - .write(reader, &key, Value::V(version), Cond::IfNoneMatch) - .await; - cluster.list(reader, "log/").await; - if slot > 3 { - let old = format!("log/{:04}", slot - 3); - let deleter = up(&cluster, rng.node()); - if cluster - .write(deleter, &old, Value::Absent, Cond::None) - .await - == WriteStatus::Acked - { - let reader = other(&cluster, deleter); - cluster.read(reader, &old, read_kind(&mut rng)).await; - cluster.list(reader, "log/").await; - } - } - } -} - -/// Reads of every key through either node, at random. -async fn random_reader(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - while Instant::now() < until { - let via = up(&cluster, rng.node()); - let pick = rng.below(6); - if pick == 5 { - let prefix = ["ptr/", "cas/", "log/"][usize::try_from(rng.below(3)).expect("small")]; - cluster.list(via, prefix).await; - continue; - } - let key = match pick { - 0 | 1 => POINTERS[usize::try_from(pick).expect("small")].to_owned(), - 2 => CAS.to_owned(), - _ => { - let slots = cluster - .history - .lock() - .unwrap() - .writes - .iter() - .filter(|write| write.key.starts_with("log/")) - .map(|write| write.key.clone()) - .next_back(); - let Some(last) = slots else { continue }; - last - } - }; - cluster.read(via, &key, read_kind(&mut rng)).await; - tokio::time::sleep(Duration::from_millis(rng.below(15))).await; - } -} - -/// Run the workload for [`RUN`] while `faults` plays (it is handed the workload's -/// deadline, and may run workload of its own), heal, let the pair serve locally again, -/// read every key through both nodes, and assert the history honours the contract and -/// both nodes agree with the origin. -async fn scenario(label: &'static str, names: [&'static str; 2], seed: u64, faults: F) -where - F: FnOnce(Arc, Instant) -> Fut, - Fut: Future + Send + 'static, -{ - // `RUST_LOG` opts a run into the nodes' decisions, each under its pod's thread. - let _ = tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "off".into()), - ) - .with_thread_names(true) - .with_test_writer() - .try_init(); - let cluster = Cluster::new(label, names, seed).await; - cluster.note("workload starts"); - let until = Instant::now() + RUN; - let tasks = vec![ - tokio::spawn(pointer_writer(Arc::clone(&cluster), seed ^ 1, until)), - tokio::spawn(cas_writer(Arc::clone(&cluster), seed ^ 2, until)), - tokio::spawn(slot_writer(Arc::clone(&cluster), seed ^ 3, until)), - tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 4, until)), - tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 5, until)), - tokio::spawn(faults(Arc::clone(&cluster), until)), - ]; - for task in tasks { - task.await.expect("workload task"); - } - cluster.net.heal(); - cluster.origin.jitter_object_responses(Duration::ZERO); - cluster.origin.fail_applied_writes(0); - cluster.note("workload ends; faults healed"); - for index in 0..2 { - if cluster.node(index).is_none() { - cluster.boot(index).await; - } - } - cluster.await_local("the run").await; - cluster.settle_abandoned().await; - let keys: Vec = { - let history = cluster.history.lock().unwrap(); - let mut keys: Vec = history - .writes - .iter() - .map(|write| write.key.clone()) - .collect(); - keys.sort(); - keys.dedup(); - keys - }; - let mut prefixes: Vec = keys - .iter() - .filter_map(|key| key.split_once('/').map(|(head, _)| format!("{head}/"))) - .collect(); - prefixes.dedup(); - for via in 0..2 { - for key in &keys { - for kind in [ReadKind::Get, ReadKind::Range, ReadKind::Head] { - cluster.read(via, key, kind).await; - } - } - for prefix in &prefixes { - cluster.list(via, prefix).await; - } - } - let mut violations = cluster.violations(); - violations.extend(cluster.divergence(&keys).await); - let report = cluster.report(&violations); - println!("{report}"); - assert!(violations.is_empty(), "{report}"); -} - -/// A cut link for longer than the lease: the writer's wait ends on a lapse. -fn cut() -> Link { - Link { - cut: true, - ..Link::default() - } -} - -/// How long a cut must last for each side to reap the other and then keep serving -/// past a stalled write's whole wait: the binary tunes `dead_timeout_ms` to the lease -/// `D` and reaps a member `2 × dead_timeout_ms` past its `Dead` verdict, which follows -/// the silence by a detection window plus the suspect timeout (well inside one more -/// `D`), and a write waits at most `D` plus a second of slack. -const PAST_THE_REAP_HORIZON: Duration = Duration::from_millis(5 * DEFAULT_LEASE_MS); - -/// Steady state, with origin answers arriving out of order. -#[tokio::test(flavor = "multi_thread")] -async fn steady_state_with_origin_jitter() { - scenario( - "rw-steady", - ["rw-steady-a", "rw-steady-b"], - 11, - |cluster, _| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(40)); - cluster.note("origin jitter 0-40ms"); - }, - ) - .await; -} - -/// Gossip delayed and lossy in both directions, below the lease. -#[tokio::test(flavor = "multi_thread")] -async fn feed_delay_and_gossip_loss() { - scenario( - "rw-lossy", - ["rw-lossy-a", "rw-lossy-b"], - 12, - |cluster, _| async move { - let [a, b] = cluster.names; - cluster.net.both( - a, - b, - Link { - delay: Duration::from_millis(150), - loss_per_mille: 300, - ..Link::default() - }, - ); - cluster.note("both directions: 150ms delay, 30% loss"); - cluster - .origin - .jitter_object_responses(Duration::from_millis(20)); - }, - ) - .await; -} - -/// Full partitions longer than the lease, then heals: writes complete on lapses, -/// and the lapsed side recovers. -#[tokio::test(flavor = "multi_thread")] -async fn partitions_longer_than_the_lease() { - scenario( - "rw-part", - ["rw-part-a", "rw-part-b"], - 13, - |cluster, _| async move { - let [a, b] = cluster.names; - for _ in 0..2 { - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.both(a, b, cut()); - cluster.note("partition a<->b"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - } - }, - ) - .await; -} - -/// One direction cut at a time, longer than the lease. -#[tokio::test(flavor = "multi_thread")] -async fn asymmetric_partitions() { - scenario( - "rw-asym", - ["rw-asym-a", "rw-asym-b"], - 14, - |cluster, _| async move { - let [a, b] = cluster.names; - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.set(a, b, cut()); - cluster.note("cut a->b"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.set(b, a, cut()); - cluster.note("cut b->a"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - }, - ) - .await; -} - -/// One direction cut for longer than the reap horizon: the reader that hears nothing -/// from the writer stops counting it as a granter while the writer still counts the -/// reader — the asymmetric partition behind a stalled acknowledgement. -#[tokio::test(flavor = "multi_thread")] -async fn an_asymmetric_partition_past_the_reap_horizon() { - scenario( - "rw-asym-reap", - ["rw-asym-reap-a", "rw-asym-reap-b"], - 21, - |cluster, _| async move { - let [a, b] = cluster.names; - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.net.set(a, b, cut()); - cluster.note("cut a->b past the reap horizon"); - tokio::time::sleep(PAST_THE_REAP_HORIZON).await; - cluster.net.heal(); - cluster.note("heal"); - }, - ) - .await; -} - -/// A pod whose only worker stalls past the lease: its lease lapses under writes. -#[tokio::test(flavor = "multi_thread")] -async fn a_stalled_pod_lapses() { - scenario( - "rw-stall", - ["rw-stall-a", "rw-stall-b"], - 15, - |cluster, _| async move { - for index in [1, 0] { - tokio::time::sleep(Duration::from_secs(3)).await; - if let Some(pod) = cluster.pods.lock().unwrap()[index].as_ref() { - pod.stall(LEASE + Duration::from_millis(800)); - } - cluster.note(format!("{} stalls", cluster.names[index])); - } - }, - ) - .await; -} - -/// A pod crashes mid-workload and comes back on its warm tier. -#[tokio::test(flavor = "multi_thread")] -async fn a_crashed_pod_restarts_on_its_warm_tier() { - scenario( - "rw-crash", - ["rw-crash-a", "rw-crash-b"], - 16, - |cluster, _| async move { - tokio::time::sleep(Duration::from_secs(3)).await; - cluster.crash(1).await; - tokio::time::sleep(Duration::from_secs(3)).await; - cluster.boot(1).await; - }, - ) - .await; -} - -/// A rolling update: each pod in turn takes the binary's planned stop and -/// restarts on its warm tier while the other serves. -#[tokio::test(flavor = "multi_thread")] -async fn a_rolling_update() { - scenario( - "rw-roll", - ["rw-roll-a", "rw-roll-b"], - 17, - |cluster, _| async move { - for index in [1, 0] { - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.planned_stop(index).await; - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.boot(index).await; - } - }, - ) - .await; -} - -/// The origin commits some writes and then fails their responses. -#[tokio::test(flavor = "multi_thread")] -async fn ambiguous_write_outcomes() { - scenario( - "rw-ambig", - ["rw-ambig-a", "rw-ambig-b"], - 18, - |cluster, _| async move { - cluster.origin.fail_applied_writes(150); - cluster - .origin - .jitter_object_responses(Duration::from_millis(20)); - cluster.note("15% of applied writes answered 500; jitter 0-20ms"); - }, - ) - .await; -} - -/// Clients that hang up mid-write while the origin answers late: an abandoned DELETE -/// or PUT still lands in every node's view of the key. -#[tokio::test(flavor = "multi_thread")] -async fn clients_that_hang_up_mid_write() { - scenario( - "rw-hangup", - ["rw-hangup-a", "rw-hangup-b"], - 19, - |cluster, until| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(60)); - cluster.note("origin jitter 0-60ms; clients give up after 0-40ms"); - let mut rng = Rng(19); - let mut n = 3_000_000; - while Instant::now() < until { - n += 1; - let key = format!("cancel/{n}"); - let via = up(&cluster, rng.node()); - if cluster.write(via, &key, Value::V(n), Cond::None).await != WriteStatus::Acked { - continue; - } - let patience = Duration::from_millis(rng.below(40)); - let other = other(&cluster, via); - cluster - .write_within(other, &key, Value::Absent, Cond::None, Some(patience)) - .await; - } - }, - ) - .await; -} - -/// Two clients overwrite one key through different nodes at once: the origin -/// decides the order, and both nodes must end up answering what it kept. -#[tokio::test(flavor = "multi_thread")] -async fn concurrent_overwrites_through_both_nodes() { - scenario( - "rw-race", - ["rw-race-a", "rw-race-b"], - 20, - |cluster, until| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(30)); - let racers = (0..2).map(|via| { - let cluster = Arc::clone(&cluster); - tokio::spawn(async move { - let mut n = 4_000_000 + via as u64 * 1_000_000; - while Instant::now() < until { - n += 1; - cluster - .write(up(&cluster, via), "race/k", Value::V(n), Cond::None) - .await; - } - }) - }); - for racer in racers.collect::>() { - racer.await.expect("racer"); - } - }, - ) - .await; -} diff --git a/tests/it/strong_rw/workload.rs b/tests/it/strong_rw/workload.rs new file mode 100644 index 0000000..fa1c352 --- /dev/null +++ b/tests/it/strong_rw/workload.rs @@ -0,0 +1,662 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::common::faultnet::Link; +use crate::common::{body_blob, request}; +use bytes::Bytes; +use s3cache::cache::proxy::CachingProxy; +use s3cache::sync::coherence::DEFAULT_LEASE_MS; +use s3cache::tier::buffer_body; +use s3s::dto::{ + DeleteObjectInput, ETag, ETagCondition, GetObjectInput, HeadObjectInput, ListObjectsV2Input, + PutObjectInput, Range, +}; +use s3s::{S3, S3Error, S3ErrorCode}; + +use super::{ + CAS, Cluster, Cond, LEASE, POINTERS, RUN, Read, ReadKind, Seen, Value, WriteStatus, version_of, +}; + +pub(super) async fn put( + proxy: &CachingProxy, + bucket: &str, + key: &str, + payload: Bytes, + if_match: Option, + if_none_match: Option, +) -> Result, S3Error> { + let version = version_of(&payload).unwrap_or_default(); + // Every fourth version names no Content-Type: the proxy indexes it skeletally + // and keeps no copy, so its first HEAD and GET are the origin's to answer. + let content_type = + (!version.is_multiple_of(4)).then(|| "application/x-s3cache-check".to_owned()); + let input = PutObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + content_length: Some(i64::try_from(payload.len()).expect("small")), + body: Some(body_blob(payload)), + content_type, + metadata: Some([("version".to_owned(), version.to_string())].into()), + if_match, + if_none_match, + ..Default::default() + }; + let out = proxy.put_object(request(input)).await?.output; + Ok(out.e_tag.map(ETag::into_value)) +} + +pub(super) async fn delete(proxy: &CachingProxy, bucket: &str, key: &str) -> Result<(), S3Error> { + let input = DeleteObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + ..Default::default() + }; + proxy.delete_object(request(input)).await?; + Ok(()) +} + +pub(super) async fn get( + proxy: &CachingProxy, + bucket: &str, + key: &str, + range: Option<(u64, u64)>, +) -> Result { + let input = GetObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + range: range.map(|(first, last)| Range::Int { + first, + last: Some(last), + }), + ..Default::default() + }; + match proxy.get_object(request(input)).await { + Ok(mut out) => { + let blob = out.output.body.take().expect("a GET carries a body"); + let bytes = buffer_body(blob, usize::MAX).await.expect("readable body"); + Ok(version_of(&bytes).map_or(Value::Absent, Value::V)) + } + Err(error) if error.code() == &S3ErrorCode::NoSuchKey => Ok(Value::Absent), + Err(error) => Err(error), + } +} + +pub(super) async fn head(proxy: &CachingProxy, bucket: &str, key: &str) -> Result { + let input = HeadObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + ..Default::default() + }; + match proxy.head_object(request(input)).await { + Ok(out) => { + let out = out.output; + Ok(Seen::Head { + etag: out.e_tag.map(ETag::into_value), + meta: out + .metadata + .as_ref() + .and_then(|meta| meta.get("version")) + .and_then(|version| version.parse().ok()), + length: out.content_length, + }) + } + Err(error) + if error.code() == &S3ErrorCode::NoSuchKey + || error.status_code() == Some(http::StatusCode::NOT_FOUND) => + { + Ok(Seen::Value(Value::Absent)) + } + Err(error) => Err(error), + } +} + +pub(super) type Rows = BTreeMap, Option)>; + +pub(super) async fn list( + proxy: &CachingProxy, + bucket: &str, + prefix: &str, +) -> Result { + let out = proxy + .list_objects_v2(request(ListObjectsV2Input { + bucket: bucket.to_owned(), + prefix: Some(prefix.to_owned()), + ..Default::default() + })) + .await? + .output; + Ok(out + .contents + .into_iter() + .flatten() + .filter_map(|object| { + Some(( + object.key?, + (object.e_tag.map(ETag::into_value), object.size), + )) + }) + .collect()) +} + +/// A seeded splitmix64 stream per workload task. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn node(&mut self) -> usize { + usize::from(self.next() % 2 == 1) + } +} + +/// The other node, unless it is down: then the same one. +fn other(cluster: &Cluster, via: usize) -> usize { + if cluster.node(1 - via).is_some() { + 1 - via + } else { + via + } +} + +/// A node that is up, preferring `via`. +fn up(cluster: &Cluster, via: usize) -> usize { + if cluster.node(via).is_some() { + via + } else { + 1 - via + } +} + +fn read_kind(rng: &mut Rng) -> ReadKind { + match rng.below(3) { + 0 => ReadKind::Get, + 1 => ReadKind::Range, + _ => ReadKind::Head, + } +} + +/// Overwrites of the pointer keys through alternating nodes, each followed by a +/// read through the other node. +async fn pointer_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut version = 0; + while Instant::now() < until { + version += 1; + let key = POINTERS[usize::try_from(version % 2).expect("small")]; + let via = up(&cluster, rng.node()); + if cluster.write(via, key, Value::V(version), Cond::None).await == WriteStatus::Acked { + let reader = other(&cluster, via); + cluster.read(reader, key, read_kind(&mut rng)).await; + if rng.below(4) == 0 { + cluster.list(reader, "ptr/").await; + } + } + } +} + +/// An `If-Match` chain: HEAD through one node, conditional overwrite through the +/// other, the way a log writer advances its pointer. +async fn cas_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut version = 1_000_000; + while Instant::now() < until { + let reader = up(&cluster, rng.node()); + let (bucket, key) = (cluster.bucket.clone(), CAS.to_owned()); + let invoked = Instant::now(); + let answer = cluster + .on(reader, move |proxy| async move { + head(&proxy, &bucket, &key).await + }) + .await + .and_then(Result::ok); + let done = Instant::now(); + let Some(current) = answer else { continue }; + cluster.history.lock().unwrap().reads.push(Read { + key: CAS.to_owned(), + op: "HEAD (CAS)", + via: reader, + invoked, + done, + seen: current.clone(), + }); + version += 1; + let cond = match current { + Seen::Head { + etag: Some(etag), .. + } => Cond::IfMatch(etag), + _ => Cond::IfNoneMatch, + }; + let writer = other(&cluster, reader); + cluster.write(writer, CAS, Value::V(version), cond).await; + } +} + +/// Log slots: create once (a contested duplicate through the other node must +/// lose), read and list through the other node, truncate old slots behind. +async fn slot_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut slot = 0_u64; + let mut version = 2_000_000; + while Instant::now() < until { + slot += 1; + let key = format!("log/{slot:04}"); + let via = up(&cluster, rng.node()); + version += 1; + if cluster + .write(via, &key, Value::V(version), Cond::IfNoneMatch) + .await + != WriteStatus::Acked + { + continue; + } + let reader = other(&cluster, via); + cluster.read(reader, &key, read_kind(&mut rng)).await; + version += 1; + cluster + .write(reader, &key, Value::V(version), Cond::IfNoneMatch) + .await; + cluster.list(reader, "log/").await; + if slot > 3 { + let old = format!("log/{:04}", slot - 3); + let deleter = up(&cluster, rng.node()); + if cluster + .write(deleter, &old, Value::Absent, Cond::None) + .await + == WriteStatus::Acked + { + let reader = other(&cluster, deleter); + cluster.read(reader, &old, read_kind(&mut rng)).await; + cluster.list(reader, "log/").await; + } + } + } +} + +/// Reads of every key through either node, at random. +async fn random_reader(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + while Instant::now() < until { + let via = up(&cluster, rng.node()); + let pick = rng.below(6); + if pick == 5 { + let prefix = ["ptr/", "cas/", "log/"][usize::try_from(rng.below(3)).expect("small")]; + cluster.list(via, prefix).await; + continue; + } + let key = match pick { + 0 | 1 => POINTERS[usize::try_from(pick).expect("small")].to_owned(), + 2 => CAS.to_owned(), + _ => { + let slots = cluster + .history + .lock() + .unwrap() + .writes + .iter() + .filter(|write| write.key.starts_with("log/")) + .map(|write| write.key.clone()) + .next_back(); + let Some(last) = slots else { continue }; + last + } + }; + cluster.read(via, &key, read_kind(&mut rng)).await; + tokio::time::sleep(Duration::from_millis(rng.below(15))).await; + } +} + +/// Run the workload for [`RUN`] while `faults` plays (it is handed the workload's +/// deadline, and may run workload of its own), heal, let the pair serve locally again, +/// read every key through both nodes, and assert the history honours the contract and +/// both nodes agree with the origin. +async fn scenario(label: &'static str, names: [&'static str; 2], seed: u64, faults: F) +where + F: FnOnce(Arc, Instant) -> Fut, + Fut: Future + Send + 'static, +{ + // `RUST_LOG` opts a run into the nodes' decisions, each under its pod's thread. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "off".into()), + ) + .with_thread_names(true) + .with_test_writer() + .try_init(); + let cluster = Cluster::new(label, names, seed).await; + cluster.note("workload starts"); + let until = Instant::now() + RUN; + let tasks = vec![ + tokio::spawn(pointer_writer(Arc::clone(&cluster), seed ^ 1, until)), + tokio::spawn(cas_writer(Arc::clone(&cluster), seed ^ 2, until)), + tokio::spawn(slot_writer(Arc::clone(&cluster), seed ^ 3, until)), + tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 4, until)), + tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 5, until)), + tokio::spawn(faults(Arc::clone(&cluster), until)), + ]; + for task in tasks { + task.await.expect("workload task"); + } + cluster.net.heal(); + cluster.origin.jitter_object_responses(Duration::ZERO); + cluster.origin.fail_applied_writes(0); + cluster.note("workload ends; faults healed"); + for index in 0..2 { + if cluster.node(index).is_none() { + cluster.boot(index).await; + } + } + cluster.await_local("the run").await; + cluster.settle_abandoned().await; + let keys: Vec = { + let history = cluster.history.lock().unwrap(); + let mut keys: Vec = history + .writes + .iter() + .map(|write| write.key.clone()) + .collect(); + keys.sort(); + keys.dedup(); + keys + }; + let mut prefixes: Vec = keys + .iter() + .filter_map(|key| key.split_once('/').map(|(head, _)| format!("{head}/"))) + .collect(); + prefixes.dedup(); + for via in 0..2 { + for key in &keys { + for kind in [ReadKind::Get, ReadKind::Range, ReadKind::Head] { + cluster.read(via, key, kind).await; + } + } + for prefix in &prefixes { + cluster.list(via, prefix).await; + } + } + let mut violations = cluster.violations(); + violations.extend(cluster.divergence(&keys).await); + let report = cluster.report(&violations); + println!("{report}"); + assert!(violations.is_empty(), "{report}"); +} + +/// A cut link for longer than the lease: the writer's wait ends on a lapse. +fn cut() -> Link { + Link { + cut: true, + ..Link::default() + } +} + +/// How long a cut must last for each side to reap the other and then keep serving +/// past a stalled write's whole wait: the binary tunes `dead_timeout_ms` to the lease +/// `D` and reaps a member `2 × dead_timeout_ms` past its `Dead` verdict, which follows +/// the silence by a detection window plus the suspect timeout (well inside one more +/// `D`), and a write waits at most `D` plus a second of slack. +const PAST_THE_REAP_HORIZON: Duration = Duration::from_millis(5 * DEFAULT_LEASE_MS); + +/// Steady state, with origin answers arriving out of order. +#[tokio::test(flavor = "multi_thread")] +async fn steady_state_with_origin_jitter() { + scenario( + "rw-steady", + ["rw-steady-a", "rw-steady-b"], + 11, + |cluster, _| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(40)); + cluster.note("origin jitter 0-40ms"); + }, + ) + .await; +} + +/// Gossip delayed and lossy in both directions, below the lease. +#[tokio::test(flavor = "multi_thread")] +async fn feed_delay_and_gossip_loss() { + scenario( + "rw-lossy", + ["rw-lossy-a", "rw-lossy-b"], + 12, + |cluster, _| async move { + let [a, b] = cluster.names; + cluster.net.both( + a, + b, + Link { + delay: Duration::from_millis(150), + loss_per_mille: 300, + ..Link::default() + }, + ); + cluster.note("both directions: 150ms delay, 30% loss"); + cluster + .origin + .jitter_object_responses(Duration::from_millis(20)); + }, + ) + .await; +} + +/// Full partitions longer than the lease, then heals: writes complete on lapses, +/// and the lapsed side recovers. +#[tokio::test(flavor = "multi_thread")] +async fn partitions_longer_than_the_lease() { + scenario( + "rw-part", + ["rw-part-a", "rw-part-b"], + 13, + |cluster, _| async move { + let [a, b] = cluster.names; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.both(a, b, cut()); + cluster.note("partition a<->b"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + } + }, + ) + .await; +} + +/// One direction cut at a time, longer than the lease. +#[tokio::test(flavor = "multi_thread")] +async fn asymmetric_partitions() { + scenario( + "rw-asym", + ["rw-asym-a", "rw-asym-b"], + 14, + |cluster, _| async move { + let [a, b] = cluster.names; + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.set(a, b, cut()); + cluster.note("cut a->b"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.set(b, a, cut()); + cluster.note("cut b->a"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + }, + ) + .await; +} + +/// One direction cut for longer than the reap horizon: the reader that hears nothing +/// from the writer stops counting it as a granter while the writer still counts the +/// reader — the asymmetric partition behind a stalled acknowledgement. +#[tokio::test(flavor = "multi_thread")] +async fn an_asymmetric_partition_past_the_reap_horizon() { + scenario( + "rw-asym-reap", + ["rw-asym-reap-a", "rw-asym-reap-b"], + 21, + |cluster, _| async move { + let [a, b] = cluster.names; + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.net.set(a, b, cut()); + cluster.note("cut a->b past the reap horizon"); + tokio::time::sleep(PAST_THE_REAP_HORIZON).await; + cluster.net.heal(); + cluster.note("heal"); + }, + ) + .await; +} + +/// A pod whose only worker stalls past the lease: its lease lapses under writes. +#[tokio::test(flavor = "multi_thread")] +async fn a_stalled_pod_lapses() { + scenario( + "rw-stall", + ["rw-stall-a", "rw-stall-b"], + 15, + |cluster, _| async move { + for index in [1, 0] { + tokio::time::sleep(Duration::from_secs(3)).await; + if let Some(pod) = cluster.pods.lock().unwrap()[index].as_ref() { + pod.stall(LEASE + Duration::from_millis(800)); + } + cluster.note(format!("{} stalls", cluster.names[index])); + } + }, + ) + .await; +} + +/// A pod crashes mid-workload and comes back on its warm tier. +#[tokio::test(flavor = "multi_thread")] +async fn a_crashed_pod_restarts_on_its_warm_tier() { + scenario( + "rw-crash", + ["rw-crash-a", "rw-crash-b"], + 16, + |cluster, _| async move { + tokio::time::sleep(Duration::from_secs(3)).await; + cluster.crash(1).await; + tokio::time::sleep(Duration::from_secs(3)).await; + cluster.boot(1).await; + }, + ) + .await; +} + +/// A rolling update: each pod in turn takes the binary's planned stop and +/// restarts on its warm tier while the other serves. +#[tokio::test(flavor = "multi_thread")] +async fn a_rolling_update() { + scenario( + "rw-roll", + ["rw-roll-a", "rw-roll-b"], + 17, + |cluster, _| async move { + for index in [1, 0] { + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.planned_stop(index).await; + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.boot(index).await; + } + }, + ) + .await; +} + +/// The origin commits some writes and then fails their responses. +#[tokio::test(flavor = "multi_thread")] +async fn ambiguous_write_outcomes() { + scenario( + "rw-ambig", + ["rw-ambig-a", "rw-ambig-b"], + 18, + |cluster, _| async move { + cluster.origin.fail_applied_writes(150); + cluster + .origin + .jitter_object_responses(Duration::from_millis(20)); + cluster.note("15% of applied writes answered 500; jitter 0-20ms"); + }, + ) + .await; +} + +/// Clients that hang up mid-write while the origin answers late: an abandoned DELETE +/// or PUT still lands in every node's view of the key. +#[tokio::test(flavor = "multi_thread")] +async fn clients_that_hang_up_mid_write() { + scenario( + "rw-hangup", + ["rw-hangup-a", "rw-hangup-b"], + 19, + |cluster, until| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(60)); + cluster.note("origin jitter 0-60ms; clients give up after 0-40ms"); + let mut rng = Rng(19); + let mut n = 3_000_000; + while Instant::now() < until { + n += 1; + let key = format!("cancel/{n}"); + let via = up(&cluster, rng.node()); + if cluster.write(via, &key, Value::V(n), Cond::None).await != WriteStatus::Acked { + continue; + } + let patience = Duration::from_millis(rng.below(40)); + let other = other(&cluster, via); + cluster + .write_within(other, &key, Value::Absent, Cond::None, Some(patience)) + .await; + } + }, + ) + .await; +} + +/// Two clients overwrite one key through different nodes at once: the origin +/// decides the order, and both nodes must end up answering what it kept. +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_overwrites_through_both_nodes() { + scenario( + "rw-race", + ["rw-race-a", "rw-race-b"], + 20, + |cluster, until| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(30)); + let racers = (0..2).map(|via| { + let cluster = Arc::clone(&cluster); + tokio::spawn(async move { + let mut n = 4_000_000 + via as u64 * 1_000_000; + while Instant::now() < until { + n += 1; + cluster + .write(up(&cluster, via), "race/k", Value::V(n), Cond::None) + .await; + } + }) + }); + for racer in racers.collect::>() { + racer.await.expect("racer"); + } + }, + ) + .await; +} From 06c11e5cc19efb8f293cb1be9063dccb6d65d7ee Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:14:00 -0600 Subject: [PATCH 03/19] build: pin groupnet to 30407c0bb057a7f12bf1520397f7398b38f97103 --- Cargo.lock | 90 ++++++++++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 2 +- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c19f7d..aee551e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1308,7 +1308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1541,11 +1541,14 @@ dependencies = [ [[package]] name = "groupnet" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "groupnet-consistency", "groupnet-core", + "groupnet-messaging", + "groupnet-network", "groupnet-runtime", + "groupnet-streams", "groupnet-transport", "groupnet-transport-mem", "groupnet-transport-tcp", @@ -1555,7 +1558,7 @@ dependencies = [ [[package]] name = "groupnet-consistency" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "bytes", "futures-util", @@ -1568,34 +1571,90 @@ dependencies = [ [[package]] name = "groupnet-core" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" + +[[package]] +name = "groupnet-messaging" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "bytes", + "groupnet-core", + "groupnet-network", + "groupnet-transport", + "ring", + "tokio", + "tokio-util", + "zerocopy", +] + +[[package]] +name = "groupnet-network" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "bytes", + "futures-util", + "groupnet-core", + "groupnet-transport", + "ring", + "rustls 0.23.45", + "tokio", + "tokio-rustls 0.26.6", + "tokio-util", + "zerocopy", +] [[package]] name = "groupnet-runtime" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "groupnet-core", + "groupnet-messaging", + "groupnet-network", + "groupnet-streams", + "groupnet-transport", + "tokio", + "tokio-util", +] + +[[package]] +name = "groupnet-streams" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", + "futures-util", "groupnet-core", + "groupnet-network", "groupnet-transport", + "ring", "tokio", + "tokio-util", + "zerocopy", ] [[package]] name = "groupnet-transport" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "bytes", "futures-util", "groupnet-core", + "ring", + "tokio", + "tokio-util", "zerocopy", ] [[package]] name = "groupnet-transport-mem" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", @@ -1605,22 +1664,26 @@ dependencies = [ [[package]] name = "groupnet-transport-tcp" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", "tokio-util", + "zerocopy", ] [[package]] name = "groupnet-transport-udp" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", + "zerocopy", ] [[package]] @@ -2822,7 +2885,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3052,7 +3115,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3122,7 +3185,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4004,6 +4067,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "libc", "pin-project-lite", "tokio", @@ -4388,7 +4452,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9f9e639..6b269c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ blake3 = "1.8.5" bytes = { version = "1", features = ["serde"] } futures = "0.3" getrandom = "0.4" -groupnet = { git = "https://github.com/napbat/groupnet", rev = "fb97452387e396c9b77e91f15c654c69489e2378", default-features = false, features = ["runtime", "consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } +groupnet = { git = "https://github.com/napbat/groupnet", rev = "30407c0bb057a7f12bf1520397f7398b38f97103", default-features = false, features = ["runtime", "consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } http = "1" http-body-util = "0.1" # Already under hyper-util/s3s; direct so the metrics endpoint can build its own responses. From 0dcc0aa51458cef83e2e04b860374f22344ca07c Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:16:35 -0600 Subject: [PATCH 04/19] refactor(sync): follow Groupnet's lease module, fallible publication and ack outcomes Lease types move to groupnet::consistency::lease and Leases::new returns a Result. WriteFeed::publish and seal report an unadvertised change instead of dropping it silently, and publish after a seal is an error rather than a panic inside Groupnet; s3cache keeps the panic. applied_by_selected takes an AckWait and returns an AckOutcome; is_complete is the old boolean, empty selection included. AckLedger::record reports a refused advertisement, now logged. The settle floor reads gossip_interval_ms, which absorbed anti_entropy_interval_ms. --- src/sync/coherence.rs | 63 +++++++++++++++++++++++++------------ src/sync/coherence/apply.rs | 30 ++++++++++-------- src/sync/stop.rs | 29 ++++++++++++----- 3 files changed, 81 insertions(+), 41 deletions(-) diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index d825667..843f9af 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -5,9 +5,11 @@ use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime}; use groupnet::consistency::volatile_recovery::{RecoveryError, RecoveryHandle, RecoveryStatus}; +use groupnet::consistency::lease::{ + CAP_LEASE, CoherenceOutcome, LeaseConfig, LeaseView, Leases, RenewalId, +}; use groupnet::consistency::{ - CAP_ACKS, CAP_LEASE, CoherenceOutcome, LeaseConfig, LeaseView, Leases, RenewalId, WriteFeed, - WriteToken, advertised_head, applied_by_selected, + AckWait, CAP_ACKS, PublishError, WriteFeed, WriteToken, advertised_head, applied_by_selected, }; use groupnet::core::volatile_bootstrap::journal::NativeCut; use groupnet::core::{NodeId, Status}; @@ -398,9 +400,10 @@ impl WriteSync { ); let capacity = NonZeroUsize::new(FEED_CAPACITY).unwrap_or(NonZeroUsize::MIN); let feed = WriteFeed::new(group.clone(), capacity, encode_event); - let leases = consistency - .leases() - .then(|| Leases::new(group.clone(), me.clone(), lease)); + let leases = consistency.leases().then(|| { + Leases::new(group.clone(), me.clone(), lease) + .expect("a validated lease config inside a Tokio runtime") + }); let lease_view = leases.as_ref().map(Leases::view); let advertisement = Advertisement::declare(group.clone(), consistency); Self { @@ -527,7 +530,7 @@ impl WriteSync { pub(crate) fn recovery_settle_floor_ms(&self) -> u64 { self.group .config() - .anti_entropy_interval_ms + .gossip_interval_ms .saturating_mul(2) } @@ -546,7 +549,7 @@ impl WriteSync { && self .lease_view .as_ref() - .is_none_or(groupnet::consistency::LeaseView::mark_caught_up) + .is_none_or(LeaseView::mark_caught_up) } /// The newest renewal of this node's serve-lease that **every** granter in its @@ -684,10 +687,21 @@ impl WriteSync { async fn advertised( &self, - publishing: impl Future, + publishing: impl Future>, metrics: &Metrics, ) -> WriteReceipt { - let token = publishing.await; + let token = match publishing.await { + Ok(token) => token, + // Recorded in the ring, which the next advertisement carries; until then no + // peer can apply it, so the write's ack wait decides whether it succeeded. + Err(PublishError::Unadvertised(unadvertised)) => { + warn!("{unadvertised}"); + unadvertised.token + } + Err(PublishError::Sealed) => { + panic!("published after the feed was sealed: the seal promised no more writes") + } + }; metrics.feed_published(); WriteReceipt { token, @@ -716,12 +730,15 @@ impl WriteSync { (Consistency::Strong | Consistency::StrongAcks, _) => { let acked = applied_by_selected( &self.group, - &self.me, - token, + AckWait { + writer: &self.me, + token, + timeout, + }, |node| waits_on(&self.group, node), - timeout, ) - .await; + .await + .is_complete(); if acked { WriteWait::Applied } else { @@ -757,13 +774,19 @@ impl WriteSync { ) -> WriteWait { let (coherent, insured) = tokio::join!( leases.invalidated_coherently(&self.me, token, self.write_wait), - applied_by_selected( - &self.group, - &self.me, - token, - |node| waits_on_unleased(&self.group, node), - timeout, - ), + async { + applied_by_selected( + &self.group, + AckWait { + writer: &self.me, + token, + timeout, + }, + |node| waits_on_unleased(&self.group, node), + ) + .await + .is_complete() + }, ); match (coherent, insured) { // A deadline the lease tier could not meet is the one outcome with no diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs index 42790f5..1d3eadc 100644 --- a/src/sync/coherence/apply.rs +++ b/src/sync/coherence/apply.rs @@ -1,6 +1,8 @@ use std::sync::Arc; -use groupnet::consistency::{AckLedger, Frontier, LeaseView, PeerWrite, PeerWrites}; +use groupnet::consistency::lease::LeaseView; +use groupnet::consistency::{AckLedger, Frontier, PeerWrite, PeerWrites, WriteToken}; +use groupnet::core::NodeId; use tracing::warn; use crate::index::KeyIndex; @@ -65,9 +67,7 @@ impl WriteSync { state.note_native_seal(native_cut(&peer, token)); sync.observe_seal(&peer, token); frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } + acknowledge(ledger.as_ref(), &peer, token).await; } PeerWrite::Renewed { peer, @@ -76,9 +76,7 @@ impl WriteSync { } => { let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); frontier.advance(&peer, start); - if let Some(ledger) = &ledger { - ledger.record(&peer, start).await; - } + acknowledge(ledger.as_ref(), &peer, start).await; } PeerWrite::Wrote { peer, @@ -102,9 +100,7 @@ impl WriteSync { // feed frontier. local.invalidate_hot(&cache_key).await; frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } + acknowledge(ledger.as_ref(), &peer, token).await; metrics.feed_applied(); // The index has moved past any copy this node holds, so the warm // copy can never serve again; left in place it would only crowd @@ -130,9 +126,7 @@ impl WriteSync { let _ = recovery.feed_gap(lapses); } frontier.advance(&peer, missed_through); - if let Some(ledger) = &ledger { - ledger.record(&peer, missed_through).await; - } + acknowledge(ledger.as_ref(), &peer, missed_through).await; metrics.feed_gap(); } } @@ -140,3 +134,13 @@ impl WriteSync { }); } } + +/// Advertise that this node applied `peer`'s feed through `token`. A refused +/// advertisement keeps the advance locally, and the next advancing record carries it. +async fn acknowledge(ledger: Option<&AckLedger>, peer: &NodeId, token: WriteToken) { + if let Some(ledger) = ledger + && let Err(error) = ledger.record(peer, token).await + { + warn!("could not advertise the ack of `{peer}` through {token:?}: {error}"); + } +} diff --git a/src/sync/stop.rs b/src/sync/stop.rs index d90fce3..0c017af 100644 --- a/src/sync/stop.rs +++ b/src/sync/stop.rs @@ -20,7 +20,7 @@ use std::time::Duration; use groupnet::consistency::volatile_recovery::{Mark, Renewal}; -use groupnet::consistency::{WriteToken, applied_by_selected}; +use groupnet::consistency::{AckWait, WriteToken, applied_by_selected}; use groupnet::core::NodeId; use tracing::{info, warn}; @@ -96,7 +96,9 @@ impl WriteSync { /// Advertise this life's feed epoch before any write, so every peer learns of the /// restart now: a sealed previous life renews, an unsealed one gaps at once. pub(crate) async fn announce(&self) { - self.feed.republish().await; + if let Err(error) = self.feed.republish().await { + warn!("could not announce this life's write feed: {error}"); + } } /// Seal this life's write feed and wait up to `wait` for the peers a write waits @@ -105,18 +107,29 @@ impl WriteSync { /// Call it only once nothing can publish again: the HTTP drain has completed and /// every mutation tail has finished. The seal promises the peers that this life wrote /// nothing after it, which is what lets them skip the restart remediation; a - /// write published afterwards panics in the feed rather than break that promise. + /// write published afterwards panics rather than break that promise. pub async fn seal(&self, wait: Duration) -> SealOutcome { - let token = self.feed.seal().await; + let token = match self.feed.seal().await { + Ok(token) => token, + // Sealed locally either way; a peer that never sees the sealed frame gaps + // after the restart, and the acknowledgement wait below reports it. + Err(unadvertised) => { + warn!("{unadvertised}"); + unadvertised.token + } + }; let observed = self.consistency.acks() && applied_by_selected( &self.group, - &self.me, - token, + AckWait { + writer: &self.me, + token, + timeout: wait, + }, |node| waits_on(&self.group, node), - wait, ) - .await; + .await + .is_complete(); if observed { info!( epoch = token.epoch, From 5f6bd1b4f37831f3d8743b6f6ea4b891e86d19f6 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:16:36 -0600 Subject: [PATCH 05/19] refactor(sync): typed seed resolution, recovery config errors and the RPITIT recovery adapter NamedSeeds takes with_* refiners and a NonZeroU32 attempt count, and the UDP transport reports its bound address directly. A link rejects a repeated neighbor, so duplicate seed ids collapse to one. RecoveryConfig carries NonZero bounds, validation names the violated rule, and the driver opens through RecoveryHandle::builder. The adapter returns AdapterError kinds and logs operation_failed. with_recovery_config returns RecoveryConfigRejected and with_recovery_rearm returns RecoveryRearmError. --- src/cache/proxy.rs | 64 +++++++++++-- src/sync/config.rs | 30 +++--- src/sync/volatile.rs | 216 ++++++++++++++++++++++--------------------- 3 files changed, 183 insertions(+), 127 deletions(-) diff --git a/src/cache/proxy.rs b/src/cache/proxy.rs index dcce9d3..21eeb21 100644 --- a/src/cache/proxy.rs +++ b/src/cache/proxy.rs @@ -414,6 +414,45 @@ impl IndexedWrite { } } +/// Why [`CachingProxy::with_recovery_config`] refused a recovery configuration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecoveryConfigRejected { + /// Groupnet's own bounds refused it. + Invalid(groupnet::core::volatile_recovery::RecoveryConfigError), + /// The settle window is shorter than two gossip rounds of this node, so a + /// renewal could still be in flight when the window closes. + SettleBelowGossip { + /// The configured settle window. + settle_ms: u64, + /// Two of this node's gossip rounds. + floor_ms: u64, + }, +} + +impl std::fmt::Display for RecoveryConfigRejected { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Invalid(error) => error.fmt(f), + Self::SettleBelowGossip { + settle_ms, + floor_ms, + } => write!( + f, + "recovery settle_ms {settle_ms} is below two gossip rounds ({floor_ms} ms)" + ), + } + } +} + +impl std::error::Error for RecoveryConfigRejected { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Invalid(error) => Some(error), + Self::SettleBelowGossip { .. } => None, + } + } +} + /// S3 service that caches LIST (from an in-memory index) and small GET/HEAD bodies (the /// hot/warm/cold [`TieredCache`]) in front of an upstream `s3s_aws::Proxy`, forwarding /// every write. @@ -505,22 +544,26 @@ impl CachingProxy { } /// Set finite recovery budgets and roster caps for gossip mode. - /// The settle window must still cover two local anti-entropy periods. + /// The settle window must still cover two local gossip rounds. /// /// # Errors - /// Returns an invalid-configuration error for zero/overflowing bounds or - /// an undersized gossip settle window. + /// Returns the rule a zero or overflowing bound breaks, or an undersized gossip + /// settle window. pub fn with_recovery_config( mut self, config: groupnet::consistency::volatile_recovery::RecoveryConfig, - ) -> Result { - config.validate()?; - if self + ) -> Result { + config.validate().map_err(RecoveryConfigRejected::Invalid)?; + if let Some(floor_ms) = self .sync .as_ref() - .is_some_and(|sync| config.settle_ms < sync.recovery_settle_floor_ms()) + .map(|sync| sync.recovery_settle_floor_ms()) + .filter(|floor_ms| config.settle_ms < *floor_ms) { - return Err(groupnet::consistency::volatile_recovery::RecoveryError::InvalidConfig); + return Err(RecoveryConfigRejected::SettleBelowGossip { + settle_ms: config.settle_ms, + floor_ms, + }); } self.recovery_config = Some(config); Ok(self) @@ -534,8 +577,9 @@ impl CachingProxy { pub fn with_recovery_rearm( mut self, policy: groupnet::consistency::volatile_recovery::RecoveryRearm, - ) -> Result { - self.recovery_rearm = Some(policy.validate()?); + ) -> Result { + policy.validate()?; + self.recovery_rearm = Some(policy); Ok(self) } diff --git a/src/sync/config.rs b/src/sync/config.rs index 96ece33..01ea34d 100644 --- a/src/sync/config.rs +++ b/src/sync/config.rs @@ -1,7 +1,8 @@ +use std::num::NonZeroU32; use std::sync::{Arc, OnceLock}; use std::time::Duration; -use groupnet::consistency::LeaseConfig; +use groupnet::consistency::lease::LeaseConfig; use groupnet::core::{Config, NodeId}; use groupnet::runtime::{Group, NamedSeeds, Node, SeedEvent, SystemResolver}; use groupnet::transport::link::{BoundLink, LinkConfig, LinkProvider}; @@ -13,7 +14,7 @@ use crate::sync::readiness::UnmetSeeds; /// Attempts to resolve a gossip seed's DNS name before it is reported /// unresolvable — a `StatefulSet` peer's record can lag its own startup. -const SEED_RESOLVE_ATTEMPTS: u32 = 30; +const SEED_RESOLVE_ATTEMPTS: NonZeroU32 = NonZeroU32::new(30).unwrap(); /// The wait between those first attempts. const SEED_RETRY: Duration = Duration::from_secs(1); @@ -28,9 +29,9 @@ const SEED_REFRESH: Duration = Duration::from_secs(15); /// s3cache's startup window and refresh cadence. fn seed_resolution() -> NamedSeeds { NamedSeeds::new(SystemResolver) - .retry_interval(SEED_RETRY) - .startup_attempts(SEED_RESOLVE_ATTEMPTS) - .refresh_interval(SEED_REFRESH) + .with_retry_interval(SEED_RETRY) + .with_startup_attempts(SEED_RESOLVE_ATTEMPTS) + .with_refresh_interval(SEED_REFRESH) } /// Logs what the seed resolver did and releases readiness for a seed that @@ -127,12 +128,15 @@ impl WriteSync { let lease = lease_config(cfg.lease_ms); let advertise = cfg .advertise - .or_else(|| transport.local_addr().ok().map(|addr| addr.to_string())); + .unwrap_or_else(|| transport.local_addr().to_string()); + // A link admits each neighbor once; a repeated seed id names the same peer. let seeds: Vec = cfg .seeds .iter() .filter(|(id, _)| *id != cfg.node_id) // a pod seeding itself (uniform config) is a no-op .map(|(id, _)| NodeId::new(id.as_str())) + .collect::>() + .into_iter() .collect(); // Seeds resolve off the startup path (DNS for a just-starting peer may // lag, and a slow resolver must not delay serving), inside groupnet: @@ -141,18 +145,16 @@ impl WriteSync { // the first `Unresolved` comes a whole startup window later. let unmet = Arc::new(OnceLock::new()); let observed = Arc::clone(&unmet); - let mut resolution = resolution.on_event(move |event| observe_seed(event, &observed)); + let mut resolution = resolution.with_observer(move |event| observe_seed(event, &observed)); for (id, addr) in &cfg.seeds { - resolution = resolution.seed(NodeId::new(id.as_str()), addr.as_str()); + resolution = resolution.with_seed(NodeId::new(id.as_str()), addr.as_str()); } - let mut builder = Node::builder(me.clone()) + let node = Node::builder(me.clone()) .link(BoundLink::new(transport, LinkConfig::new(seeds.clone()))) .config(gossip_config(cfg.lease_ms)) - .named_seeds(resolution); - if let Some(advertise) = advertise { - builder = builder.advertise_addr(advertise); - } - let node = match builder.start().await { + .named_seeds(resolution) + .advertise_addr(advertise); + let node = match node.start().await { Ok(node) => node, Err(error) => { warn!("gossip disabled: cannot start managed network: {error}"); diff --git a/src/sync/volatile.rs b/src/sync/volatile.rs index d8b9a6f..155b16f 100644 --- a/src/sync/volatile.rs +++ b/src/sync/volatile.rs @@ -1,16 +1,18 @@ //! Application facts and origin-index effects for Groupnet's volatile recovery. use std::collections::BTreeSet; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; +use groupnet::consistency::lease::CAP_LEASE; use groupnet::consistency::{ - CAP_LEASE, FrontierView, WriteToken, checked_advertised_head, + FrontierView, WriteToken, checked_advertised_head, volatile_recovery::{ - AdapterError, BoxRecoveryFuture, Mark, Peer, PeerObservation, PublicationPermit, - RecoveryAdapter, RecoveryConfig, RecoveryFallback, RecoveryHandle, RecoveryMode, - RecoveryOperation, RecoveryRearm, RecoveryStage, + AdapterError, AdapterErrorKind, Mark, Peer, PeerObservation, PublicationPermit, + RecoveryAdapter, RecoveryBuilder, RecoveryConfig, RecoveryFallback, RecoveryHandle, + RecoveryMode, RecoveryOperation, RecoveryRearm, RecoveryStage, }, }; use groupnet::core::{NodeId, Status}; @@ -125,9 +127,9 @@ impl WriteSync { .unwrap_or(2_000) .max(self.recovery_settle_floor_ms()); let config = config.unwrap_or(RecoveryConfig { - max_members: 256, - max_member_bytes: 256, - max_barrier_rounds: 4, + max_members: const { NonZeroUsize::new(256).unwrap() }, + max_member_bytes: const { NonZeroUsize::new(256).unwrap() }, + max_barrier_rounds: const { NonZeroU32::new(4).unwrap() }, // Neither bound has to fit a bucket's scan time. Every committed LIST page // reports progress, which restarts both, so a scan that keeps committing // pages runs to completion in one pass. `attempt_ms` is how long a scan may @@ -160,26 +162,25 @@ struct PreparedRecovery { } impl PreparedRecovery { - fn open_origin(&self) -> RecoveryHandle { - if let Some(policy) = self.rearm { - RecoveryHandle::open_with_rearm( - Arc::clone(&self.adapter), - self.config, - self.mode, - self.adapter.me.clone(), - self.session, - policy, - ) - } else { - RecoveryHandle::open( - Arc::clone(&self.adapter), - self.config, - self.mode, - self.adapter.me.clone(), - self.session, - ) + /// The driver's builder with this node's configuration, mode, and rearm policy. + fn builder(&self) -> RecoveryBuilder { + let builder = RecoveryHandle::builder( + Arc::clone(&self.adapter), + self.adapter.me.clone(), + self.session, + ) + .config(self.config) + .mode(self.mode); + match self.rearm { + Some(policy) => builder.rearm(policy), + None => builder, } - .expect("valid recovery configuration and Tokio runtime") + } + + fn open_origin(&self) -> RecoveryHandle { + self.builder() + .open() + .expect("valid recovery configuration and Tokio runtime") } } @@ -198,17 +199,20 @@ fn observe_group( lease: Duration, limits: RecoveryConfig, ) -> PeerObservation { - let bound = limits.max_members.checked_add(1).ok_or(AdapterError)?; + let bound = limits + .max_members + .checked_add(1) + .ok_or(AdapterErrorKind::Capacity)?; let roster = group .statuses_held_bounded(bound, limits.max_member_bytes) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; let mut peers = Vec::with_capacity(roster.len().saturating_sub(1)); for (node, status, held) in roster { if node == *me { continue; } let head = checked_advertised_head(group, &node) - .map_err(|_| AdapterError)? + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))? .map(|token| Mark { epoch: token.epoch, sequence: token.seq, @@ -245,103 +249,99 @@ impl RecoveryAdapter for CacheRecoveryAdapter { } } + fn operation_failed(&self, stage: RecoveryStage, error: AdapterError) { + let cause = std::error::Error::source(&error).map(ToString::to_string); + tracing::warn!(?stage, ?cause, "recovery operation failed: {error}"); + } + fn fell_back(&self, from: RecoveryStage, reason: RecoveryFallback) { self.metrics.recovery_fallback(); tracing::info!(?from, ?reason, "recovery fell back"); } - fn invalidate( + async fn invalidate( &self, _op: RecoveryOperation, distrust_bodies: bool, permit: PublicationPermit, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { - permit - .publish(|| { - if distrust_bodies { - self.local.distrust_all(); - } - }) - .ok_or(AdapterError) - }) + ) -> Result<(), AdapterError> { + permit + .publish(|| { + if distrust_bodies { + self.local.distrust_all(); + } + }) + .ok_or_else(fenced) } - fn rebuild_origin( + async fn rebuild_origin( &self, _op: RecoveryOperation, permit: PublicationPermit, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { - self.metrics.recovery_origin_scan(); - let buckets: BTreeSet = { - let state = self.state.read().map_err(|_| AdapterError)?; - if state.len() > MAX_RECOVERY_BUCKETS - || state.len().saturating_add(self.buckets.len()) > MAX_RECOVERY_BUCKETS - { - return Err(AdapterError); - } - let mut buckets: BTreeSet = self.buckets.iter().cloned().collect(); - buckets.extend(state.keys().cloned()); - buckets - }; - let mut generations = Vec::with_capacity(buckets.len()); - for bucket in buckets { - let generation = permit - .publish(|| begin_bucket_resync(&self.state, &bucket)) - .ok_or(AdapterError)?; - generations.push((bucket, generation)); - } - for (bucket, generation) in generations { - sync_bucket_generation_guarded( - &self.client, - &self.state, - &bucket, - generation, - self.scan, - permit.clone(), - ) - .await - .map_err(|_| AdapterError)?; + ) -> Result<(), AdapterError> { + self.metrics.recovery_origin_scan(); + let buckets: BTreeSet = { + let state = self + .state + .read() + .map_err(|_| AdapterError::new(AdapterErrorKind::Unavailable))?; + if state.len() > MAX_RECOVERY_BUCKETS + || state.len().saturating_add(self.buckets.len()) > MAX_RECOVERY_BUCKETS + { + return Err(AdapterErrorKind::Capacity.into()); } - Ok(()) - }) + let mut buckets: BTreeSet = self.buckets.iter().cloned().collect(); + buckets.extend(state.keys().cloned()); + buckets + }; + let mut generations = Vec::with_capacity(buckets.len()); + for bucket in buckets { + let generation = permit + .publish(|| begin_bucket_resync(&self.state, &bucket)) + .ok_or_else(fenced)?; + generations.push((bucket, generation)); + } + for (bucket, generation) in generations { + sync_bucket_generation_guarded( + &self.client, + &self.state, + &bucket, + generation, + self.scan, + permit.clone(), + ) + .await + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error))?; + } + Ok(()) } - fn observe_peers( - &self, - _op: RecoveryOperation, - limits: RecoveryConfig, - ) -> BoxRecoveryFuture<'_, PeerObservation> { - Box::pin(async move { - let sync = self.sync.upgrade().ok_or(AdapterError)?; - observe_group(&self.group, &sync, &self.me, self.lease, limits) - }) + async fn observe_peers(&self, _op: RecoveryOperation, limits: RecoveryConfig) -> PeerObservation { + let sync = self.sync.upgrade().ok_or_else(closed)?; + observe_group(&self.group, &sync, &self.me, self.lease, limits) } - fn wait_frontiers( + async fn wait_frontiers( &self, _op: RecoveryOperation, heads: Vec<(NodeId, Mark)>, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { - let frontier = self.frontier().ok_or(AdapterError)?; - for (node, head) in heads { - if !frontier - .reached( - &node, - WriteToken { - epoch: head.epoch, - seq: head.sequence, - }, - ) - .await - { - return Err(AdapterError); - } + ) -> Result<(), AdapterError> { + let frontier = self.frontier().ok_or_else(closed)?; + for (node, head) in heads { + if !frontier + .reached( + &node, + WriteToken { + epoch: head.epoch, + seq: head.sequence, + }, + ) + .await + { + return Err(AdapterErrorKind::Unavailable.into()); } - Ok(()) - }) + } + Ok(()) } fn affirm(&self, _op: RecoveryOperation) -> bool { @@ -351,6 +351,16 @@ impl RecoveryAdapter for CacheRecoveryAdapter { } } +/// The operation's publication permit is no longer current. +fn fenced() -> AdapterError { + AdapterError::new(AdapterErrorKind::Fenced) +} + +/// The write feed this adapter serves has shut down or never started its apply loop. +fn closed() -> AdapterError { + AdapterError::new(AdapterErrorKind::Unavailable) +} + #[cfg(test)] mod tests { use super::*; From 06642fa2910488f7d267d58327ba38f3657603f2 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:21:11 -0600 Subject: [PATCH 06/19] refactor(sync): port the fleet bootstrap adapter to Groupnet's typed bootstrap ports Bootstrap ports are RPITIT traits, so the adapter implements async fns. Every journal, transfer, claim, wire and admission bound is NonZero; s3cache's own image accounting stays usize and converts at the Groupnet boundary. Effects run under one EffectContext, the journal takes a CapturePlan, ImageSize and NativeRenewal, NativeClaimSource and BulkDonorPort take option structs, the session takes a PresenceIdentity, and the driver opens through the recovery builder. Adapter failures carry an AdapterErrorKind. A refused feed retirement advertisement stays local until the next publish, as before. --- src/index/fleet.rs | 10 +- src/index/fleet/capture.rs | 38 ++++-- src/sync/coherence.rs | 2 +- src/sync/volatile/fleet.rs | 27 ++-- src/sync/volatile/fleet/capture.rs | 60 +++++---- src/sync/volatile/fleet/donor.rs | 194 ++++++++++++++-------------- src/sync/volatile/fleet/follower.rs | 84 ++++++------ src/sync/volatile/fleet/runtime.rs | 114 ++++++++-------- 8 files changed, 280 insertions(+), 249 deletions(-) diff --git a/src/index/fleet.rs b/src/index/fleet.rs index 56cbb14..5cacf3d 100644 --- a/src/index/fleet.rs +++ b/src/index/fleet.rs @@ -8,7 +8,9 @@ use std::collections::{BTreeMap, HashMap}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use groupnet::consistency::volatile_recovery::bootstrap::ports::{JournalIngress, LogicalClock}; -use groupnet::core::volatile_bootstrap::journal::{DeltaIdentity, Invalidation, NativeCut}; +use groupnet::core::volatile_bootstrap::journal::{ + DeltaIdentity, Invalidation, NativeCut, NativeRenewal, +}; use s3s::dto::{ETag, ObjectStorageClass}; @@ -165,9 +167,13 @@ impl IndexCapture { return; }; let now = self.clock.now(); + let renewal = NativeRenewal { + sealed: sealed.clone(), + epoch, + }; let _ = self .ingress - .with_journal(|journal| journal.renew(now, self.generation, sealed, epoch, effect)); + .with_journal(|journal| journal.renew(now, self.generation, renewal, effect)); } } diff --git a/src/index/fleet/capture.rs b/src/index/fleet/capture.rs index ceb4058..c486408 100644 --- a/src/index/fleet/capture.rs +++ b/src/index/fleet/capture.rs @@ -1,6 +1,7 @@ //! Exact C publication under the live index lock in O(buckets); measuring and //! encoding the image run off-lock, from the snapshot taken at C. +use std::num::NonZeroUsize; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -10,7 +11,9 @@ use groupnet::consistency::volatile_recovery::bootstrap::admission::{ use groupnet::consistency::volatile_recovery::bootstrap::ports::{ DonorCapture, JournalIngress, LogicalClock, }; -use groupnet::core::volatile_bootstrap::journal::{DonorJournal, Invalidation, NativeCut}; +use groupnet::core::volatile_bootstrap::journal::{ + CapturePlan, DonorJournal, ImageSize as JournalImageSize, Invalidation, NativeCut, +}; use groupnet::core::volatile_bootstrap::{BootstrapMemberIdentity, same_membership}; use tokio::sync::Notify; @@ -126,8 +129,12 @@ impl PendingFleetCapture { // The clock every local effect was journaled on, so finishing can never // land behind a write that arrived while the image was encoding. let now = capture.clock().now(); + let actual = JournalImageSize { + encoded_bytes: NonZeroUsize::new(image.len()).ok_or(ImageError::Incomplete)?, + decoded_bytes: NonZeroUsize::new(private_size).ok_or(ImageError::Incomplete)?, + }; self.ingress - .with_journal(|journal| journal.finish_capture(now, image.len(), private_size)) + .with_journal(|journal| journal.finish_capture(now, actual)) .map_err(|_| ImageError::Incomplete)?; } let encoded = self.encoded.take().ok_or(ImageError::Corrupt)?; @@ -193,20 +200,24 @@ impl KeyIndex { // complete sync, no uncertainty, and the ceilings before anything is // reserved; the rows themselves are measured off-lock at encode. let size = tallied_size(&live, universe, caps)?; + let image = JournalImageSize { + encoded_bytes: NonZeroUsize::new(size.encoded_bytes).ok_or(ImageError::Capacity)?, + decoded_bytes: NonZeroUsize::new(size.decoded_bytes).ok_or(ImageError::Capacity)?, + }; let encoded = admission - .reserve(AdmissionClass::Encoded, size.encoded_bytes) + .reserve(AdmissionClass::Encoded, image.encoded_bytes) .map_err(|_| ImageError::Capacity)?; let decoded = admission - .reserve(AdmissionClass::Decoded, size.decoded_bytes) + .reserve(AdmissionClass::Decoded, image.decoded_bytes) .map_err(|_| ImageError::Capacity)?; let snapshot = snapshot_state(&live, universe)?; let mut cuts = Vec::new(); - if live.native_cuts.len() > config.max_cuts { + if live.native_cuts.len() > config.max_cuts.get() { return Err(ImageError::Capacity); } cuts.try_reserve_exact(live.native_cuts.len()) .map_err(|_| ImageError::Capacity)?; - if cuts.capacity() > config.max_cuts.saturating_mul(2).max(4) { + if cuts.capacity() > config.max_cuts.get().saturating_mul(2).max(4) { return Err(ImageError::Capacity); } let mut cut_bytes = 0usize; @@ -214,14 +225,14 @@ impl KeyIndex { cut_bytes = cut_bytes .checked_add(writer.len()) .ok_or(ImageError::Capacity)?; - if writer.is_empty() || cut_bytes > config.max_cut_bytes { + if writer.is_empty() || cut_bytes > config.max_cut_bytes.get() { return Err(ImageError::Capacity); } let mut identity = Vec::new(); identity .try_reserve_exact(writer.len()) .map_err(|_| ImageError::Capacity)?; - if identity.capacity() > config.max_cut_bytes { + if identity.capacity() > config.max_cut_bytes.get() { return Err(ImageError::Capacity); } identity.extend_from_slice(writer); @@ -234,10 +245,11 @@ impl KeyIndex { journal .begin_capture( clock.now(), - size.encoded_bytes, - size.decoded_bytes, - members, - cuts, + CapturePlan { + image, + members, + cuts, + }, ) .map_err(|_| ImageError::Incomplete)?; let ingress = @@ -249,7 +261,7 @@ impl KeyIndex { ingress.clone(), generation, clock, - config.max_event_bytes, + config.max_event_bytes.get(), caps.name_bytes, )); let lock_held = locked.elapsed(); diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index 843f9af..d31c616 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -871,7 +871,7 @@ impl WriteSync { // `retire_through` retains the current head as an anchor and re-advertises the // compacted window; a dropped advertisement is carried by the next publish. if retires_feed { - self.feed.retire_through(token).await; + let _ = self.feed.retire_through(token).await; } guaranteed } diff --git a/src/sync/volatile/fleet.rs b/src/sync/volatile/fleet.rs index 111d1eb..f290dbf 100644 --- a/src/sync/volatile/fleet.rs +++ b/src/sync/volatile/fleet.rs @@ -1,5 +1,6 @@ //! S3 index facts supplied to Groupnet's one optional bootstrap worker. +use std::num::NonZeroUsize; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -16,7 +17,7 @@ use groupnet::core::volatile_bootstrap::{ use crate::index::fleet::FleetDonorImage; -use super::{AdapterError, CacheRecoveryAdapter, RecoveryAdapter}; +use super::{AdapterError, AdapterErrorKind, CacheRecoveryAdapter, RecoveryAdapter, closed, fenced}; mod capture; mod donor; @@ -25,9 +26,9 @@ mod runtime; pub(crate) use runtime::FleetListenerGuard; pub(super) use runtime::open_recovery; -const MAX_ROSTER: usize = 64; -const MAX_ROSTER_ID_BYTES: usize = 256; -const MAX_ROSTER_METADATA: usize = 64 << 10; +const MAX_ROSTER: NonZeroUsize = NonZeroUsize::new(64).unwrap(); +const MAX_ROSTER_ID_BYTES: NonZeroUsize = NonZeroUsize::new(256).unwrap(); +const MAX_ROSTER_METADATA: NonZeroUsize = NonZeroUsize::new(64 << 10).unwrap(); /// Consumer index/source evidence. Claim election, bulk retry, and lifetime /// decisions remain in the existing Groupnet recovery worker. @@ -87,7 +88,7 @@ impl FleetStatePort { || universe.len() > capture::IMAGE_CAPS.buckets || universe.windows(2).any(|pair| pair[0] >= pair[1]) { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } Ok(Self { adapter, @@ -138,7 +139,7 @@ impl FleetStatePort { // cause a second origin LIST. Once local recovery reaches Ready, the // same Groupnet worker attempts one finite guarded index recapture. if !request.permit.valid() { - return Err(AdapterError); + return Err(fenced()); } Ok(LocalCaptureOutcome::LocalOnly) } @@ -174,7 +175,7 @@ impl FleetStatePort { request: ReadyCaptureRequest, admission: &ByteAdmission, ) -> Result, AdapterError> { - let sync = self.adapter.sync.upgrade().ok_or(AdapterError)?; + let sync = self.adapter.sync.upgrade().ok_or_else(closed)?; if !self.listener_alive() || !self .source_matches( @@ -186,7 +187,7 @@ impl FleetStatePort { .await { tracing::info!("fleet Ready recapture declined: source roster changed"); - return Err(AdapterError); + return Err(fenced()); } let started = Instant::now(); let id = capture::next_id( @@ -214,7 +215,7 @@ impl FleetStatePort { || !sync.mode_allows_local() || !self.listener_alive() { - return Err(AdapterError); + return Err(fenced()); } prepared.attach( &self.adapter.state, @@ -225,7 +226,7 @@ impl FleetStatePort { request.wake, ) }) - .ok_or(AdapterError) + .ok_or_else(fenced) .flatten() .inspect_err(|_| { tracing::info!( @@ -241,7 +242,7 @@ impl FleetStatePort { .await { tracing::info!("fleet Ready recapture declined: roster changed while encoding"); - return Err(AdapterError); + return Err(fenced()); } let captured = request .guard @@ -250,11 +251,11 @@ impl FleetStatePort { || !sync.mode_allows_local() || !self.listener_alive() { - return Err(AdapterError); + return Err(fenced()); } capture::finish(pending) }) - .ok_or(AdapterError) + .ok_or_else(fenced) .flatten(); if captured.is_ok() { tracing::info!( diff --git a/src/sync/volatile/fleet/capture.rs b/src/sync/volatile/fleet/capture.rs index 2891f0c..c234079 100644 --- a/src/sync/volatile/fleet/capture.rs +++ b/src/sync/volatile/fleet/capture.rs @@ -1,5 +1,6 @@ //! Finite donor C capture, with all large encoding outside the index lock. +use std::num::NonZeroUsize; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -16,7 +17,7 @@ use tokio::sync::Notify; use crate::index::KeyIndex; use crate::index::fleet::{FleetDonorImage, ImageCaps, PendingFleetCapture}; -use super::super::AdapterError; +use super::super::{AdapterError, AdapterErrorKind, fenced}; static NEXT_CAPTURE: AtomicU64 = AtomicU64::new(1); @@ -35,23 +36,28 @@ pub(super) const IMAGE_CAPS: ImageCaps = ImageCaps { name_bytes: 1_024, }; +/// `n` as a bound Groupnet accepts; a zero bound fails to compile. +pub(super) const fn bound(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).expect("journal bounds are nonzero") +} + pub(super) const JOURNAL_CONFIG: JournalConfig = JournalConfig { - max_encoded_bytes: IMAGE_CAPS.bytes, - max_decoded_bytes: IMAGE_CAPS.decoded_bytes, - max_events: 1_024, - max_suffix_bytes: 8 << 20, - max_event_bytes: 4_096, - max_identity_bytes: 256, - max_followers: 2, - max_follower_id_bytes: 256, - max_cuts: 64, - max_cut_bytes: 4_096, - max_members: 64, - max_membership_bytes: 16 << 10, - max_scope_bytes: 8_192, - max_batch_events: 32, - max_batch_bytes: 128 << 10, - max_inflight_bytes: 256 << 10, + max_encoded_bytes: bound(IMAGE_CAPS.bytes), + max_decoded_bytes: bound(IMAGE_CAPS.decoded_bytes), + max_events: bound(1_024), + max_suffix_bytes: bound(8 << 20), + max_event_bytes: bound(4_096), + max_identity_bytes: bound(256), + max_followers: bound(2), + max_follower_id_bytes: bound(256), + max_cuts: bound(64), + max_cut_bytes: bound(4_096), + max_members: bound(64), + max_membership_bytes: bound(16 << 10), + max_scope_bytes: bound(8_192), + max_batch_events: bound(32), + max_batch_bytes: bound(128 << 10), + max_inflight_bytes: bound(256 << 10), max_total_ms: 600_000, // A 108 MB production image is about 1,700 sequential 64 KiB chunks; give a // follower on a loaded node minutes, not one, before its reservation expires. @@ -67,9 +73,9 @@ pub(super) fn next_id( .try_update(Ordering::AcqRel, Ordering::Acquire, |old| { old.checked_add(1) }) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; if serial == 0 || generation == 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } Ok(CaptureId { scope, @@ -91,11 +97,13 @@ pub(super) fn prepare( budget: &ByteAdmission, id: CaptureId, ) -> Result { - let suffix_bytes = DonorJournal::storage_bound(JOURNAL_CONFIG).map_err(|_| AdapterError)?; + let suffix_bytes = DonorJournal::storage_bound(JOURNAL_CONFIG) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::InvalidInput, error))?; let suffix = budget .reserve(AdmissionClass::Suffix, suffix_bytes) - .map_err(|_| AdapterError)?; - let journal = DonorJournal::new(JOURNAL_CONFIG, id).map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; + let journal = DonorJournal::new(JOURNAL_CONFIG, id) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::InvalidInput, error))?; Ok(PreparedCapture { journal, suffix }) } @@ -122,7 +130,7 @@ impl PreparedCapture { IMAGE_CAPS, clock, ) - .map_err(|_| AdapterError) + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity)) } } @@ -136,17 +144,17 @@ pub(super) async fn encode( tokio::task::spawn_blocking(move || { pending .encode(&universe, IMAGE_CAPS) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; Ok(pending) }) .await - .map_err(|_| AdapterError)? + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error))? } pub(super) fn finish( pending: PendingFleetCapture, ) -> Result, AdapterError> { - pending.finish().map_err(|_| AdapterError) + pending.finish().map_err(|_| fenced()) } pub(super) fn same_roster( diff --git a/src/sync/volatile/fleet/donor.rs b/src/sync/volatile/fleet/donor.rs index 375cb1b..3b0bb89 100644 --- a/src/sync/volatile/fleet/donor.rs +++ b/src/sync/volatile/fleet/donor.rs @@ -1,27 +1,36 @@ //! Groupnet donor callbacks: complete guarded C and immutable bounded chunks. use std::mem::size_of; -use std::time::Instant; +use std::num::NonZeroUsize; use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionClass, Admitted, ByteAdmission, }; use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::BootstrapStatePort; use groupnet::consistency::volatile_recovery::bootstrap::ports::{ - DonorCapture, LocalCaptureOutcome, LocalCaptureRequest, ReadyCaptureRequest, TransferContext, + DonorCapture, EffectContext, LocalCaptureOutcome, LocalCaptureRequest, ReadyCaptureRequest, TransferResources, }; -use groupnet::consistency::volatile_recovery::{ - AdapterError, BoxRecoveryFuture, PublicationPermit, -}; +use groupnet::consistency::volatile_recovery::{AdapterError, AdapterErrorKind}; use groupnet::core::volatile_bootstrap::journal::{AttachToken, JournalBatch, JournalCursor}; use groupnet::core::volatile_bootstrap::transfer::{TransferEffect, TransferEvent, TransferOffer}; use crate::index::fleet::{FleetDonorImage, FleetStage}; +use crate::sync::volatile::fenced; use super::{FleetStatePort, capture}; -const CHUNK_BYTES: usize = 64 << 10; +const CHUNK_BYTES: NonZeroUsize = NonZeroUsize::new(64 << 10).unwrap(); + +/// A request or argument outside this donor's bounds. +fn invalid() -> AdapterError { + AdapterError::new(AdapterErrorKind::InvalidInput) +} + +/// An arithmetic bound or memory admission refused the work. +fn capacity() -> AdapterError { + AdapterError::new(AdapterErrorKind::Capacity) +} impl BootstrapStatePort for FleetStatePort { type Image = FleetDonorImage; @@ -31,20 +40,20 @@ impl BootstrapStatePort for FleetStatePort { // refuses any concurrent final effect at the guarded install. type NativeBuffer = (); - fn build_local_capture<'a>( - &'a self, + async fn build_local_capture( + &self, request: LocalCaptureRequest, - admission: &'a ByteAdmission, - ) -> BoxRecoveryFuture<'a, Result, AdapterError>> { - Box::pin(self.initial_capture(request, admission)) + admission: &ByteAdmission, + ) -> Result, AdapterError> { + self.initial_capture(request, admission).await } - fn recapture_current_index<'a>( - &'a self, + async fn recapture_current_index( + &self, request: ReadyCaptureRequest, - admission: &'a ByteAdmission, - ) -> BoxRecoveryFuture<'a, Result, AdapterError>> { - Box::pin(self.ready_capture(request, admission)) + admission: &ByteAdmission, + ) -> Result, AdapterError> { + self.ready_capture(request, admission).await } fn retire_local_capture(&self, capture: &DonorCapture) { @@ -54,15 +63,15 @@ impl BootstrapStatePort for FleetStatePort { fn image_offer( &self, capture: &DonorCapture, - max_metadata_bytes: usize, + max_metadata_bytes: NonZeroUsize, admission: &ByteAdmission, ) -> Result, AdapterError> { if !capture.is_active() || !capture::still_attached(&self.adapter.state, capture) { - return Err(AdapterError); + return Err(fenced()); } - let bytes = capture.image().as_bytes().len(); - if bytes == 0 || bytes > capture::IMAGE_CAPS.bytes || max_metadata_bytes == 0 { - return Err(AdapterError); + let bytes = NonZeroUsize::new(capture.image().as_bytes().len()).ok_or_else(invalid)?; + if bytes.get() > capture::IMAGE_CAPS.bytes { + return Err(capacity()); } let chunks = bytes.div_ceil(CHUNK_BYTES); // Charge the complete retained DTO, both scope copies, cut vectors, @@ -70,27 +79,27 @@ impl BootstrapStatePort for FleetStatePort { // entering the journal and allocating any response-owned metadata. let headers = size_of::() .checked_add( - capture::JOURNAL_CONFIG.max_members + capture::JOURNAL_CONFIG.max_members.get() * size_of::(), ) .and_then(|n| { n.checked_add( - capture::JOURNAL_CONFIG.max_cuts + capture::JOURNAL_CONFIG.max_cuts.get() * size_of::(), ) }) - .ok_or(AdapterError)?; + .ok_or_else(capacity)?; let charge = max_metadata_bytes - .checked_mul(4) + .checked_mul(NonZeroUsize::new(4).ok_or_else(capacity)?) .and_then(|n| n.checked_add(headers)) - .ok_or(AdapterError)?; + .ok_or_else(capacity)?; let permit = admission .reserve(AdmissionClass::Inflight, charge) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; // The decoded charge finished at C is the measured image charge the // follower must reserve and decode under. let offer = capture.ingress().with_journal(|journal| { - let decoded_bytes = journal.image_charge()?.decoded_bytes; + let decoded_bytes = journal.image_charge()?.image.decoded_bytes; let id = journal.id().clone(); let members = journal.image_members().to_vec(); let cuts = journal.image_cuts().to_vec(); @@ -109,7 +118,7 @@ impl BootstrapStatePort for FleetStatePort { cuts, }) }); - let offer = offer.ok_or(AdapterError)?; + let offer = offer.ok_or_else(fenced)?; let variable = offer .capture .scope @@ -128,9 +137,12 @@ impl BootstrapStatePort for FleetStatePort { .iter() .try_fold(n, |sum, cut| sum.checked_add(cut.writer.len())) }) - .ok_or(AdapterError)?; - if variable > max_metadata_bytes || !capture.is_active() { - return Err(AdapterError); + .ok_or_else(capacity)?; + if variable > max_metadata_bytes.get() { + return Err(capacity()); + } + if !capture.is_active() { + return Err(fenced()); } Ok(permit.hold(offer)) } @@ -139,97 +151,87 @@ impl BootstrapStatePort for FleetStatePort { &self, capture: &DonorCapture, sequence: usize, - max_bytes: usize, + max_bytes: NonZeroUsize, admission: &ByteAdmission, ) -> Result>, AdapterError> { - if !capture.is_active() - || !capture::still_attached(&self.adapter.state, capture) - || max_bytes == 0 - || max_bytes > CHUNK_BYTES - { - return Err(AdapterError); + if !capture.is_active() || !capture::still_attached(&self.adapter.state, capture) { + return Err(fenced()); } - let start = sequence.checked_mul(CHUNK_BYTES).ok_or(AdapterError)?; + if max_bytes > CHUNK_BYTES { + return Err(invalid()); + } + let start = sequence.checked_mul(CHUNK_BYTES.get()).ok_or_else(invalid)?; let end = start - .checked_add(CHUNK_BYTES) - .ok_or(AdapterError)? + .checked_add(CHUNK_BYTES.get()) + .ok_or_else(invalid)? .min(capture.image().as_bytes().len()); let slice = capture .image() .as_bytes() .get(start..end) - .ok_or(AdapterError)?; - if slice.is_empty() || slice.len() > max_bytes { - return Err(AdapterError); + .ok_or_else(invalid)?; + if slice.is_empty() || slice.len() > max_bytes.get() { + return Err(invalid()); } let permit = admission .reserve(AdmissionClass::Encoded, max_bytes) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; let mut copy = Vec::new(); copy.try_reserve_exact(slice.len()) - .map_err(|_| AdapterError)?; - if copy.capacity() > max_bytes { - return Err(AdapterError); + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; + if copy.capacity() > max_bytes.get() { + return Err(capacity()); } copy.extend_from_slice(slice); Ok(permit.hold(copy)) } - fn execute_local<'a>( - &'a self, - context: &'a TransferContext, + async fn execute_local( + &self, effect: TransferEffect, - resources: &'a mut TransferResources, - admission: &'a ByteAdmission, - permit: Option, - deadline: Instant, - ) -> BoxRecoveryFuture<'a, Result>, AdapterError>> { - Box::pin(self.execute_follower(context, effect, resources, admission, permit, deadline)) + resources: &mut TransferResources, + context: EffectContext<'_>, + ) -> Result>, AdapterError> { + self.execute_follower(effect, resources, context).await } - fn store_chunk<'a>( - &'a self, + async fn store_chunk( + &self, sequence: usize, - chunk: &'a Admitted>, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result> { - Box::pin(async move { - let stage = resources.stage.as_mut().ok_or(AdapterError)?; - stage - .stage_mut() - .store_chunk(sequence, chunk.get()) - .map_err(|_| AdapterError)?; - Ok(stage.reserved_bytes().1) - }) + chunk: &Admitted>, + resources: &mut TransferResources, + ) -> Result { + let stage = resources.stage.as_mut().ok_or_else(fenced)?; + stage + .stage_mut() + .store_chunk(sequence, chunk.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; + Ok(stage.reserved_bytes().1) } - fn stage_batch<'a>( - &'a self, - batch: &'a Admitted, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result<(), AdapterError>> { - Box::pin(async move { - resources - .stage - .as_mut() - .ok_or(AdapterError)? - .stage_mut() - .stage_batch(batch.get()) - .map_err(|_| AdapterError) - }) + async fn stage_batch( + &self, + batch: &Admitted, + resources: &mut TransferResources, + ) -> Result<(), AdapterError> { + resources + .stage + .as_mut() + .ok_or_else(fenced)? + .stage_mut() + .stage_batch(batch.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed)) } - fn detach<'a>( - &'a self, + async fn detach( + &self, token: AttachToken, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result<(), AdapterError>> { - Box::pin(async move { - if resources.attachment.as_ref() != Some(&token) { - return Err(AdapterError); - } - resources.attachment.take(); - Ok(()) - }) + resources: &mut TransferResources, + ) -> Result<(), AdapterError> { + if resources.attachment.as_ref() != Some(&token) { + return Err(fenced()); + } + resources.attachment.take(); + Ok(()) } } diff --git a/src/sync/volatile/fleet/follower.rs b/src/sync/volatile/fleet/follower.rs index b9b7dec..e474943 100644 --- a/src/sync/volatile/fleet/follower.rs +++ b/src/sync/volatile/fleet/follower.rs @@ -1,6 +1,7 @@ //! Private follower staging and one guarded native-feed/index handoff. use std::mem::size_of; +use std::num::NonZeroUsize; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; @@ -8,15 +9,16 @@ use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionClass, Admitted, ByteAdmission, Reservation, }; use groupnet::consistency::volatile_recovery::bootstrap::ports::{ - StageResources, TransferContext, TransferResources, + EffectContext, StageResources, TransferResources, }; -use groupnet::consistency::volatile_recovery::{AdapterError, PublicationPermit}; +use groupnet::consistency::volatile_recovery::{AdapterError, AdapterErrorKind}; use groupnet::core::volatile_bootstrap::journal::AttachToken; use groupnet::core::volatile_bootstrap::transfer::{ NativeCoverageReceipt, NativeHandoffReceipt, TransferEffect, TransferEvent, }; use crate::index::fleet::{FleetStage, IMAGE_SCHEMA, InstallRefusal}; +use crate::sync::volatile::fenced; use super::{FleetStatePort, capture}; @@ -31,16 +33,16 @@ impl FleetStatePort { // B, proven cuts, membership, and handoff metadata coexist with // the original effect and retained stage. Charge this physical // result copy separately from Groupnet's core event envelope. - (64_usize << 10) - .checked_mul(4) + NonZeroUsize::new(64_usize << 10) + .and_then(|n| n.checked_mul(NonZeroUsize::new(4)?)) .and_then(|n| n.checked_add(size_of::())) - .ok_or(AdapterError)? + .ok_or(AdapterErrorKind::Capacity)? } else { - size_of::().max(1) + NonZeroUsize::new(size_of::()).unwrap_or(NonZeroUsize::MIN) }; admission .reserve(AdmissionClass::Inflight, bytes) - .map_err(|_| AdapterError) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error)) } #[expect( @@ -49,15 +51,19 @@ impl FleetStatePort { )] pub(super) async fn execute_follower( &self, - context: &TransferContext, effect: TransferEffect, resources: &mut TransferResources, - admission: &ByteAdmission, - permit: Option, - deadline: Instant, + context: EffectContext<'_>, ) -> Result>, AdapterError> { + let EffectContext { + transfer: context, + admission, + permit, + deadline, + .. + } = context; if Instant::now() >= deadline { - return Err(AdapterError); + return Err(AdapterErrorKind::Expired.into()); } match effect { TransferEffect::ReserveStage { @@ -68,15 +74,15 @@ impl FleetStatePort { decoded_bytes, } => { if resources.stage.is_some() - || encoded_bytes > capture::IMAGE_CAPS.bytes - || decoded_bytes > capture::IMAGE_CAPS.decoded_bytes + || encoded_bytes.get() > capture::IMAGE_CAPS.bytes + || decoded_bytes.get() > capture::IMAGE_CAPS.decoded_bytes { tracing::debug!( encoded_bytes, decoded_bytes, "fleet peer image declined: over the image codec ceilings" ); - return Err(AdapterError); + return Err(AdapterErrorKind::Capacity.into()); } let reserved = admission .reserve(AdmissionClass::Encoded, encoded_bytes) @@ -85,30 +91,30 @@ impl FleetStatePort { .reserve(AdmissionClass::Decoded, decoded_bytes) .map(|decoded| (encoded, decoded)) }); - let Ok((encoded, decoded)) = reserved else { + let (encoded, decoded) = reserved.map_err(|error| { tracing::debug!( encoded_bytes, decoded_bytes, "fleet peer image declined: memory admission" ); - return Err(AdapterError); - }; + AdapterError::with_source(AdapterErrorKind::Capacity, error) + })?; let stage = FleetStage::new( - encoded_bytes, - chunks, + encoded_bytes.get(), + chunks.get(), image_cut, - decoded_bytes, + decoded_bytes.get(), capture::IMAGE_CAPS, - capture::JOURNAL_CONFIG.max_event_bytes, + capture::JOURNAL_CONFIG.max_event_bytes.get(), ) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::InvalidInput))?; resources.stage = Some(StageResources::new(stage, encoded, decoded)?); Ok(Some( Self::event_charge(admission, false)?.hold(TransferEvent::StageReserved { op }), )) } TransferEffect::VerifyImage { op, commitment } => { - let mut stage = resources.stage.take().ok_or(AdapterError)?; + let mut stage = resources.stage.take().ok_or_else(fenced)?; // Hash and full map decode are bounded but potentially large. // The blocking task owns the stage and both permits until it // exits, even if the recovery worker is cancelled. @@ -116,11 +122,11 @@ impl FleetStatePort { stage .stage_mut() .verify(commitment) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; Ok::<_, AdapterError>(stage) }) .await - .map_err(|_| AdapterError) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error)) .flatten() .inspect_err(|_| { tracing::debug!("fleet peer image declined: commitment or decode failed"); @@ -142,9 +148,9 @@ impl FleetStatePort { .source_matches(op, &receipt.members, admission, deadline) .await { - return Err(AdapterError); + return Err(fenced()); } - let stage = resources.stage.as_mut().ok_or(AdapterError)?; + let stage = resources.stage.as_mut().ok_or_else(fenced)?; match stage.stage_mut().check_coverage( &self.adapter.state, &receipt.covered_cuts, @@ -165,7 +171,7 @@ impl FleetStatePort { refusal = %why, "fleet peer image declined: live index incompatible" ); - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } } let charge = Self::event_charge(admission, true)?; @@ -182,15 +188,15 @@ impl FleetStatePort { )) } TransferEffect::InstallCandidate { op, coverage } => { - let permit = permit.ok_or(AdapterError)?.restricted_to(deadline); - let attachment = resources.attachment.as_ref().ok_or(AdapterError)?; + let permit = permit.ok_or_else(fenced)?.restricted_to(deadline); + let attachment = resources.attachment.as_ref().ok_or_else(fenced)?; if coverage.parent != context.parent || coverage.barrier.reservation != attachment.reservation || coverage.barrier.attach_operation != attachment.operation || coverage.staged_through != coverage.barrier.cursor || coverage.buffered_bytes != 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } // Membership can change after the B coverage observation even // when the local index publication serial stays unchanged. @@ -199,7 +205,7 @@ impl FleetStatePort { .source_matches(op, &coverage.members, admission, deadline) .await { - return Err(AdapterError); + return Err(fenced()); } let charge = Self::event_charge(admission, true)?; let continued_cuts = coverage.proven_cuts.clone(); @@ -208,11 +214,11 @@ impl FleetStatePort { .try_update(Ordering::AcqRel, Ordering::Acquire, |old| { old.checked_add(1) }) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; if applier_generation == 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::Capacity.into()); } - let stage = resources.stage.as_mut().ok_or(AdapterError)?; + let stage = resources.stage.as_mut().ok_or_else(fenced)?; let installed = permit .publish(|| { stage.stage_mut().install_into( @@ -221,7 +227,7 @@ impl FleetStatePort { &self.universe, ) }) - .ok_or(AdapterError)?; + .ok_or_else(fenced)?; match installed { // The image moved into the live index; release the // emptied stage and its private memory permits. @@ -251,7 +257,7 @@ impl FleetStatePort { refusal = %why, "fleet peer image declined at install: live index incompatible" ); - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } } let handoff = NativeHandoffReceipt { @@ -274,7 +280,7 @@ impl FleetStatePort { resources.native_overlap.take(); Ok(None) } - _ => Err(AdapterError), + _ => Err(AdapterErrorKind::Unsupported.into()), } } } diff --git a/src/sync/volatile/fleet/runtime.rs b/src/sync/volatile/fleet/runtime.rs index 39e57ae..398e8e3 100644 --- a/src/sync/volatile/fleet/runtime.rs +++ b/src/sync/volatile/fleet/runtime.rs @@ -1,17 +1,23 @@ //! One TCP listener and claim/transfer child of the recovery worker. +use std::num::NonZeroU32; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use groupnet::consistency::volatile_recovery::RecoveryHandle; use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionLimits, ByteAdmission, }; -use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::BulkDonorPort; +use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::{ + BulkDonorOptions, BulkDonorPort, +}; use groupnet::consistency::volatile_recovery::bootstrap::bulk_wire::{ BootstrapBulkListener, BulkLimits, PhaseLimits, WireLimits, }; -use groupnet::consistency::volatile_recovery::bootstrap::native_claims::NativeClaimSource; +use groupnet::consistency::volatile_recovery::bootstrap::native_claims::{ + NativeClaimOptions, NativeClaimSource, +}; use groupnet::consistency::volatile_recovery::bootstrap::ports::BootstrapCapabilities; use groupnet::consistency::volatile_recovery::bootstrap::report::{ BootstrapDecision, BootstrapObserver, @@ -19,9 +25,10 @@ use groupnet::consistency::volatile_recovery::bootstrap::report::{ use groupnet::consistency::volatile_recovery::bootstrap::session::{ BootstrapRuntimeConfig, BootstrapSession, }; +use groupnet::consistency::QueueCapacity; use groupnet::core::NodeId; use groupnet::core::volatile_bootstrap::transfer::TransferConfig; -use groupnet::core::volatile_bootstrap::{BootstrapConfig, BootstrapScope}; +use groupnet::core::volatile_bootstrap::{BootstrapConfig, BootstrapScope, PresenceIdentity}; use groupnet::transport::bulk::DataPlane; use groupnet::transport::tcp::TcpBulkTransport; use tokio::sync::watch; @@ -31,6 +38,7 @@ use crate::metrics::Metrics; use crate::sync::coherence::WriteSync; use crate::sync::fleet::fresh_boot; +use super::capture::bound; use super::{FleetStatePort, capture}; use crate::sync::volatile::{CacheRecoveryAdapter, PreparedRecovery}; @@ -97,7 +105,7 @@ fn claim_config(recovery: &PreparedRecovery) -> Option { let total_ms = recovery.config.total_ms.min(60_000); (total_ms >= 5_000).then_some(BootstrapConfig { max_members: capture::JOURNAL_CONFIG.max_members, - max_member_bytes: 256, + max_member_bytes: bound(256), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, // Give a newly observed Ready donor's native TTL claim time to // propagate during the existing core settle phase. This remains @@ -111,36 +119,36 @@ fn claim_config(recovery: &PreparedRecovery) -> Option { }) } -fn transfer_config() -> TransferConfig { - TransferConfig { - expected_schema: crate::index::fleet::IMAGE_SCHEMA, - max_metadata_bytes: 64 << 10, - max_encoded_bytes: capture::IMAGE_CAPS.bytes, - max_decoded_bytes: capture::IMAGE_CAPS.decoded_bytes, - max_chunk_bytes: 64 << 10, - max_chunks: capture::IMAGE_CAPS.bytes / (64 << 10), +fn transfer_config() -> Option { + Some(TransferConfig { + expected_schema: NonZeroU32::new(crate::index::fleet::IMAGE_SCHEMA)?, + max_metadata_bytes: bound(64 << 10), + max_encoded_bytes: bound(capture::IMAGE_CAPS.bytes), + max_decoded_bytes: bound(capture::IMAGE_CAPS.decoded_bytes), + max_chunk_bytes: bound(64 << 10), + max_chunks: bound(capture::IMAGE_CAPS.bytes / (64 << 10)), max_batch_bytes: capture::JOURNAL_CONFIG.max_batch_bytes, max_batch_events: capture::JOURNAL_CONFIG.max_batch_events, max_replay_events: capture::JOURNAL_CONFIG.max_events, - max_native_buffer_bytes: 1 << 20, + max_native_buffer_bytes: bound(1 << 20), max_members: capture::JOURNAL_CONFIG.max_members, max_cuts: capture::JOURNAL_CONFIG.max_cuts, coverage_poll_ms: 100, - } + }) } fn bulk_limits() -> BulkLimits { BulkLimits { wire: WireLimits { - max_frame_bytes: 256 << 10, + max_frame_bytes: bound(256 << 10), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, - max_node_bytes: 256, - max_payload_bytes: 192 << 10, + max_node_bytes: bound(256), + max_payload_bytes: bound(192 << 10), }, phase: PhaseLimits { - max_body_bytes: 128 << 10, + max_body_bytes: bound(128 << 10), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, - max_node_bytes: 256, + max_node_bytes: bound(256), max_writer_bytes: capture::JOURNAL_CONFIG.max_cut_bytes, max_cuts: capture::JOURNAL_CONFIG.max_cuts, max_members: capture::JOURNAL_CONFIG.max_members, @@ -148,7 +156,7 @@ fn bulk_limits() -> BulkLimits { max_identity_bytes: capture::JOURNAL_CONFIG.max_identity_bytes, max_effect_bytes: capture::JOURNAL_CONFIG.max_event_bytes, }, - server_request_ms: 3_000, + server_request: Duration::from_millis(3_000), } } @@ -157,13 +165,13 @@ fn bulk_limits() -> BulkLimits { /// safety cap. One node is a donor or a follower within a recovery, never both. fn admission() -> Option { ByteAdmission::new(AdmissionLimits { - max_total_bytes: (256 << 20) + (1 << 30) + (80 << 20), + max_total_bytes: bound((256 << 20) + (1 << 30) + (80 << 20)), max_encoded_bytes: capture::IMAGE_CAPS.bytes, max_decoded_bytes: capture::IMAGE_CAPS.decoded_bytes, max_suffix_bytes: 32 << 20, max_native_overlap_bytes: 16 << 20, max_inflight_bytes: 32 << 20, - max_reservations: 512, + max_reservations: bound(512), }) .ok() } @@ -192,15 +200,17 @@ pub(in crate::sync::volatile) async fn open_recovery( let fleet = prepared.fleet.as_ref()?; let scope: BootstrapScope = required("scope", fleet.scope(&prepared.adapter.buckets).ok())?; let claim = required("claim configuration", claim_config(prepared))?; - let claim = required("claim policy", claim.validate().ok())?; - let transfer = required("transfer policy", transfer_config().validate().ok())?; - let limits = required("bulk limits", bulk_limits().validate().ok())?; + required("claim policy", claim.validate().ok())?; + let transfer = required("transfer schema", transfer_config())?; + required("transfer policy", transfer.validate().ok())?; + let limits = bulk_limits(); + required("bulk limits", limits.validate().ok())?; let budget = required("admission", admission())?; let suffix_bytes = groupnet::core::volatile_bootstrap::journal::DonorJournal::storage_bound( capture::JOURNAL_CONFIG, ) .ok()?; - if suffix_bytes > (32 << 20) { + if suffix_bytes.get() > (32 << 20) { tracing::warn!("fleet suffix exceeds shared admission cap; using guarded origin recovery"); return None; } @@ -225,11 +235,13 @@ pub(in crate::sync::volatile) async fn open_recovery( "native claim source", NativeClaimSource::new( sync.group.clone(), - scope.clone(), - claim, - 32 << 10, - 64 << 10, budget.clone(), + NativeClaimOptions { + scope: scope.clone(), + policy: claim, + max_key_bytes: bound(32 << 10), + max_value_bytes: bound(64 << 10), + }, ) .ok(), )?); @@ -251,9 +263,11 @@ pub(in crate::sync::volatile) async fn open_recovery( Arc::clone(&state), plane.clone(), budget.clone(), - scope.clone(), - transfer, - limits, + BulkDonorOptions { + scope: scope.clone(), + policy: transfer, + limits, + }, ) .ok(), )?); @@ -265,8 +279,8 @@ pub(in crate::sync::volatile) async fn open_recovery( let runtime_config = BootstrapRuntimeConfig { claim, transfer, - max_claim_metadata_bytes: 64 << 10, - donor_inbox_capacity: 32, + max_claim_metadata_bytes: bound(64 << 10), + donor_inbox_capacity: required("donor inbox", QueueCapacity::new(32))?, require_participation: true, }; let (child, sender) = required( @@ -275,9 +289,11 @@ pub(in crate::sync::volatile) async fn open_recovery( capabilities, runtime_config, scope, - sync.me.clone(), - boot, - prepared.session, + PresenceIdentity { + node: sync.me.clone(), + boot, + session: prepared.session, + }, ) .ok(), )?; @@ -287,27 +303,7 @@ pub(in crate::sync::volatile) async fn open_recovery( )?; let bootstrap = Box::new(child.with_observer(Arc::new(DecisionLog(Arc::clone(&prepared.adapter.metrics))))); - let handle = if let Some(rearm) = prepared.rearm { - RecoveryHandle::open_with_bootstrap_and_rearm( - Arc::clone(&prepared.adapter), - prepared.config, - prepared.mode, - sync.me.clone(), - prepared.session, - rearm, - bootstrap, - ) - } else { - RecoveryHandle::open_with_bootstrap( - Arc::clone(&prepared.adapter), - prepared.config, - prepared.mode, - sync.me.clone(), - prepared.session, - bootstrap, - ) - } - .ok()?; + let handle = prepared.builder().bootstrap(bootstrap).open().ok()?; let (shutdown, receiver) = watch::channel(false); let task = tokio::spawn(async move { if let Err(error) = listener.run(receiver).await { From cbb77c19d5ba2ed5d01eaeaf91f6113b68f35ccf Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:24:38 -0600 Subject: [PATCH 07/19] refactor(sync): satisfy the adapter lints and rustfmt after the Groupnet port The synchronous adapter operations return ready futures: an async fn without an await trips unused_async_trait_impl, and an async block trips manual_async_fn. --- src/sync/coherence.rs | 7 +--- src/sync/volatile.rs | 43 +++++++++++--------- src/sync/volatile/fleet.rs | 4 +- src/sync/volatile/fleet/donor.rs | 65 ++++++++++++++++++------------ src/sync/volatile/fleet/runtime.rs | 2 +- 5 files changed, 71 insertions(+), 50 deletions(-) diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index d31c616..49c0d28 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -4,10 +4,10 @@ use std::num::NonZeroUsize; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime}; -use groupnet::consistency::volatile_recovery::{RecoveryError, RecoveryHandle, RecoveryStatus}; use groupnet::consistency::lease::{ CAP_LEASE, CoherenceOutcome, LeaseConfig, LeaseView, Leases, RenewalId, }; +use groupnet::consistency::volatile_recovery::{RecoveryError, RecoveryHandle, RecoveryStatus}; use groupnet::consistency::{ AckWait, CAP_ACKS, PublishError, WriteFeed, WriteToken, advertised_head, applied_by_selected, }; @@ -528,10 +528,7 @@ impl WriteSync { } pub(crate) fn recovery_settle_floor_ms(&self) -> u64 { - self.group - .config() - .gossip_interval_ms - .saturating_mul(2) + self.group.config().gossip_interval_ms.saturating_mul(2) } pub(super) fn frontier_view(&self) -> Option { diff --git a/src/sync/volatile.rs b/src/sync/volatile.rs index 155b16f..72ff44f 100644 --- a/src/sync/volatile.rs +++ b/src/sync/volatile.rs @@ -259,19 +259,21 @@ impl RecoveryAdapter for CacheRecoveryAdapter { tracing::info!(?from, ?reason, "recovery fell back"); } - async fn invalidate( + fn invalidate( &self, _op: RecoveryOperation, distrust_bodies: bool, permit: PublicationPermit, - ) -> Result<(), AdapterError> { - permit - .publish(|| { - if distrust_bodies { - self.local.distrust_all(); - } - }) - .ok_or_else(fenced) + ) -> impl Future> + Send { + std::future::ready( + permit + .publish(|| { + if distrust_bodies { + self.local.distrust_all(); + } + }) + .ok_or_else(fenced), + ) } async fn rebuild_origin( @@ -316,9 +318,15 @@ impl RecoveryAdapter for CacheRecoveryAdapter { Ok(()) } - async fn observe_peers(&self, _op: RecoveryOperation, limits: RecoveryConfig) -> PeerObservation { - let sync = self.sync.upgrade().ok_or_else(closed)?; - observe_group(&self.group, &sync, &self.me, self.lease, limits) + fn observe_peers( + &self, + _op: RecoveryOperation, + limits: RecoveryConfig, + ) -> impl Future + Send { + std::future::ready(match self.sync.upgrade() { + Some(sync) => observe_group(&self.group, &sync, &self.me, self.lease, limits), + None => Err(closed()), + }) } async fn wait_frontiers( @@ -364,7 +372,7 @@ fn closed() -> AdapterError { #[cfg(test)] mod tests { use super::*; - use groupnet::consistency::LeaseConfig; + use groupnet::consistency::lease::LeaseConfig; use groupnet::core::Config; use groupnet::network::RouterConfig; use groupnet::runtime::Node; @@ -377,8 +385,7 @@ mod tests { let a = NodeId::new("recovery-a"); let b = NodeId::new("recovery-b"); let config = Config { - gossip_interval_ms: 10, - anti_entropy_interval_ms: 25, + gossip_interval_ms: 25, ..Config::default() }; let routing = RouterConfig { @@ -422,9 +429,9 @@ mod tests { None, ); let limits = RecoveryConfig { - max_members: 4, - max_member_bytes: 64, - max_barrier_rounds: 2, + max_members: NonZeroUsize::new(4).unwrap(), + max_member_bytes: NonZeroUsize::new(64).unwrap(), + max_barrier_rounds: NonZeroU32::new(2).unwrap(), total_ms: 2_000, attempt_ms: 500, settle_ms: 100, diff --git a/src/sync/volatile/fleet.rs b/src/sync/volatile/fleet.rs index f290dbf..f18028a 100644 --- a/src/sync/volatile/fleet.rs +++ b/src/sync/volatile/fleet.rs @@ -17,7 +17,9 @@ use groupnet::core::volatile_bootstrap::{ use crate::index::fleet::FleetDonorImage; -use super::{AdapterError, AdapterErrorKind, CacheRecoveryAdapter, RecoveryAdapter, closed, fenced}; +use super::{ + AdapterError, AdapterErrorKind, CacheRecoveryAdapter, RecoveryAdapter, closed, fenced, +}; mod capture; mod donor; diff --git a/src/sync/volatile/fleet/donor.rs b/src/sync/volatile/fleet/donor.rs index 3b0bb89..cdbcfc6 100644 --- a/src/sync/volatile/fleet/donor.rs +++ b/src/sync/volatile/fleet/donor.rs @@ -160,7 +160,9 @@ impl BootstrapStatePort for FleetStatePort { if max_bytes > CHUNK_BYTES { return Err(invalid()); } - let start = sequence.checked_mul(CHUNK_BYTES.get()).ok_or_else(invalid)?; + let start = sequence + .checked_mul(CHUNK_BYTES.get()) + .ok_or_else(invalid)?; let end = start .checked_add(CHUNK_BYTES.get()) .ok_or_else(invalid)? @@ -195,43 +197,56 @@ impl BootstrapStatePort for FleetStatePort { self.execute_follower(effect, resources, context).await } - async fn store_chunk( + fn store_chunk( &self, sequence: usize, chunk: &Admitted>, resources: &mut TransferResources, - ) -> Result { - let stage = resources.stage.as_mut().ok_or_else(fenced)?; - stage - .stage_mut() - .store_chunk(sequence, chunk.get()) - .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; - Ok(stage.reserved_bytes().1) + ) -> impl Future> + Send { + std::future::ready( + resources + .stage + .as_mut() + .ok_or_else(fenced) + .and_then(|stage| { + stage + .stage_mut() + .store_chunk(sequence, chunk.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; + Ok(stage.reserved_bytes().1) + }), + ) } - async fn stage_batch( + fn stage_batch( &self, batch: &Admitted, resources: &mut TransferResources, - ) -> Result<(), AdapterError> { - resources - .stage - .as_mut() - .ok_or_else(fenced)? - .stage_mut() - .stage_batch(batch.get()) - .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed)) + ) -> impl Future> + Send { + std::future::ready( + resources + .stage + .as_mut() + .ok_or_else(fenced) + .and_then(|stage| { + stage + .stage_mut() + .stage_batch(batch.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed)) + }), + ) } - async fn detach( + fn detach( &self, token: AttachToken, resources: &mut TransferResources, - ) -> Result<(), AdapterError> { - if resources.attachment.as_ref() != Some(&token) { - return Err(fenced()); - } - resources.attachment.take(); - Ok(()) + ) -> impl Future> + Send { + std::future::ready(if resources.attachment.as_ref() == Some(&token) { + resources.attachment.take(); + Ok(()) + } else { + Err(fenced()) + }) } } diff --git a/src/sync/volatile/fleet/runtime.rs b/src/sync/volatile/fleet/runtime.rs index 398e8e3..aae34f3 100644 --- a/src/sync/volatile/fleet/runtime.rs +++ b/src/sync/volatile/fleet/runtime.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use groupnet::consistency::QueueCapacity; use groupnet::consistency::volatile_recovery::RecoveryHandle; use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionLimits, ByteAdmission, @@ -25,7 +26,6 @@ use groupnet::consistency::volatile_recovery::bootstrap::report::{ use groupnet::consistency::volatile_recovery::bootstrap::session::{ BootstrapRuntimeConfig, BootstrapSession, }; -use groupnet::consistency::QueueCapacity; use groupnet::core::NodeId; use groupnet::core::volatile_bootstrap::transfer::TransferConfig; use groupnet::core::volatile_bootstrap::{BootstrapConfig, BootstrapScope, PresenceIdentity}; From 44f59b03a47c8e5ea751f44579f45c406f7f7922 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:24:38 -0600 Subject: [PATCH 08/19] test: port unit and integration fixtures to the Groupnet 30407c0 API Lease types come from consistency::lease, publish returns a Result, seed refiners are with_*, RecoveryConfig and journal bounds are NonZero, and the fault transport fills Inbound::source with None because it keeps no address book. The test configs that set anti_entropy_interval_ms to 25 next to a 10 ms gossip_interval_ms now set gossip_interval_ms to 25: the engine scheduled digest rounds on the former, and Groupnet folded the two into one cadence. --- src/cache/tests.rs | 8 +++--- src/index/fleet/capture.rs | 55 +++++++++++++++++++++---------------- src/index/fleet/stage.rs | 2 +- src/sync/tests.rs | 19 +++++++------ tests/it/common/faultnet.rs | 2 ++ tests/it/fleet_bootstrap.rs | 19 +++++++------ tests/it/startup.rs | 7 +++-- 7 files changed, 62 insertions(+), 50 deletions(-) diff --git a/src/cache/tests.rs b/src/cache/tests.rs index a4af8be..ce0f2a7 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use bytes::Bytes; -use groupnet::consistency::{LeaseConfig, WriteFeed, advertised_head}; +use groupnet::consistency::lease::LeaseConfig; +use groupnet::consistency::{WriteFeed, advertised_head}; use groupnet::core::{Config, NodeId}; use groupnet::network::RouterConfig; use groupnet::runtime::Node; @@ -27,12 +28,11 @@ use crate::tier::CachedObject; /// warm-up window is milliseconds rather than a second. fn brisk() -> Config { Config { - gossip_interval_ms: 10, + gossip_interval_ms: 25, probe_interval_ms: 20, probe_timeout_ms: 10, suspect_timeout_ms: 50, dead_timeout_ms: 300, - anti_entropy_interval_ms: 25, ..Config::default() } } @@ -401,7 +401,7 @@ async fn a_freshness_timeout_routes_to_origin_and_is_counted() { NonZeroUsize::new(4).unwrap_or(NonZeroUsize::MIN), |_value: &u8| vec![0], ); - let token = feed.publish(&1).await; + let token = feed.publish(&1).await.expect("the feed is open"); tokio::time::timeout(Duration::from_secs(2), async { loop { if advertised_head(&b_group, &a_id) == Some(token) { diff --git a/src/index/fleet/capture.rs b/src/index/fleet/capture.rs index c486408..b62a483 100644 --- a/src/index/fleet/capture.rs +++ b/src/index/fleet/capture.rs @@ -336,24 +336,28 @@ mod tests { use super::*; + fn n(value: usize) -> NonZeroUsize { + NonZeroUsize::new(value).unwrap() + } + fn config() -> JournalConfig { JournalConfig { - max_encoded_bytes: 2_048, - max_decoded_bytes: 8_192, - max_events: 4, - max_suffix_bytes: 1_024, - max_event_bytes: 128, - max_identity_bytes: 64, - max_followers: 1, - max_follower_id_bytes: 64, - max_cuts: 2, - max_cut_bytes: 128, - max_members: 2, - max_membership_bytes: 256, - max_scope_bytes: 128, - max_batch_events: 1, - max_batch_bytes: 128, - max_inflight_bytes: 256, + max_encoded_bytes: n(2_048), + max_decoded_bytes: n(8_192), + max_events: n(4), + max_suffix_bytes: n(1_024), + max_event_bytes: n(128), + max_identity_bytes: n(64), + max_followers: n(1), + max_follower_id_bytes: n(64), + max_cuts: n(2), + max_cut_bytes: n(128), + max_members: n(2), + max_membership_bytes: n(256), + max_scope_bytes: n(128), + max_batch_events: n(1), + max_batch_bytes: n(128), + max_inflight_bytes: n(256), max_total_ms: 100_000, max_follower_ms: 10_000, } @@ -391,13 +395,13 @@ mod tests { fn admission() -> ByteAdmission { ByteAdmission::new(AdmissionLimits { - max_total_bytes: 1_000_000, + max_total_bytes: n(1_000_000), max_encoded_bytes: 16_384, max_decoded_bytes: 65_536, max_suffix_bytes: 900_000, max_native_overlap_bytes: 1, max_inflight_bytes: 1, - max_reservations: 16, + max_reservations: n(16), }) .unwrap() } @@ -483,7 +487,10 @@ mod tests { donor .ingress() .with_journal(|journal| journal.image_charge()) - .map(|charge| (charge.encoded_bytes, charge.decoded_bytes)), + .map(|charge| ( + charge.image.encoded_bytes.get(), + charge.image.decoded_bytes.get() + )), Some((size.encoded_bytes, size.decoded_bytes)) ); } @@ -550,7 +557,7 @@ mod tests { DonorJournal::storage_bound(config).unwrap(), ) .unwrap(); - let held = budget.reserve(AdmissionClass::Decoded, 65_536).unwrap(); + let held = budget.reserve(AdmissionClass::Decoded, n(65_536)).unwrap(); let before = budget.usage().0; let refused = index.begin_fleet_capture( journal, @@ -886,18 +893,18 @@ mod tests { name_bytes: 64, }; let config = JournalConfig { - max_encoded_bytes: caps.bytes, - max_decoded_bytes: caps.decoded_bytes, + max_encoded_bytes: n(caps.bytes), + max_decoded_bytes: n(caps.decoded_bytes), ..config() }; let budget = ByteAdmission::new(AdmissionLimits { - max_total_bytes: 1 << 28, + max_total_bytes: n(1 << 28), max_encoded_bytes: caps.bytes, max_decoded_bytes: caps.decoded_bytes, max_suffix_bytes: 900_000, max_native_overlap_bytes: 1, max_inflight_bytes: 1, - max_reservations: 16, + max_reservations: n(16), }) .unwrap(); let suffix = budget diff --git a/src/index/fleet/stage.rs b/src/index/fleet/stage.rs index 6273f54..37c5419 100644 --- a/src/index/fleet/stage.rs +++ b/src/index/fleet/stage.rs @@ -335,7 +335,7 @@ mod tests { identity: DeltaIdentity::Local(b"repair".to_vec()), effect, }], - bytes: 42, + bytes: std::num::NonZeroUsize::new(42).unwrap(), }; stage.stage_batch(&batch).unwrap(); assert_eq!(stage.through(), &through); diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 2ac1005..5c0878d 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -1,8 +1,10 @@ +use std::num::NonZeroU32; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; +use groupnet::consistency::CAP_ACKS; +use groupnet::consistency::lease::{CAP_LEASE, LeaseConfig}; use groupnet::consistency::volatile_recovery::{RecoveryStage, RecoveryStatus}; -use groupnet::consistency::{CAP_ACKS, CAP_LEASE, LeaseConfig}; use groupnet::core::volatile_recovery::RecoveryState; use groupnet::core::{Config, NodeId, Status}; use groupnet::runtime::{NamedSeeds, ResolveFuture, SeedResolver, SystemResolver}; @@ -72,12 +74,11 @@ fn a_reopened_recovery_generation_cannot_authorize_an_old_local_answer() { /// reap-bounded behaviour is the same shape production gets, only faster. fn brisk() -> Config { Config { - gossip_interval_ms: 10, + gossip_interval_ms: 25, probe_interval_ms: 20, probe_timeout_ms: 10, suspect_timeout_ms: 50, dead_timeout_ms: 300, - anti_entropy_interval_ms: 25, ..Config::default() } } @@ -200,7 +201,7 @@ fn env_spellings_parse_into_a_config() { const TEST_SEED_CADENCE: Duration = Duration::from_millis(20); /// Failed lookups before the test's unresolvable seed is reported. -const TEST_STARTUP_ATTEMPTS: u32 = 3; +const TEST_STARTUP_ATTEMPTS: NonZeroU32 = NonZeroU32::new(3).unwrap(); /// How long a loopback gossip pair may take to (re)meet. const TEST_REJOIN: Duration = Duration::from_secs(20); @@ -261,9 +262,9 @@ async fn within(limit: Duration, what: &str, mut cond: impl FnMut() -> bool) { #[tokio::test(flavor = "multi_thread")] async fn an_unresolvable_seed_releases_readiness_after_its_window() { let resolution = NamedSeeds::new(MovableSeed::default()) - .retry_interval(TEST_SEED_CADENCE) - .startup_attempts(TEST_STARTUP_ATTEMPTS) - .refresh_interval(TEST_SEED_CADENCE); + .with_retry_interval(TEST_SEED_CADENCE) + .with_startup_attempts(TEST_STARTUP_ATTEMPTS) + .with_refresh_interval(TEST_SEED_CADENCE); let sync = WriteSync::bind( seed_config("seed-lone", loopback(), &[("seed-ghost", "ghost:1")]), resolution, @@ -289,8 +290,8 @@ async fn a_moved_seed_is_relearned_and_rejoins() { let name_b = MovableSeed::default(); name_b.point_at(Some(b_first)); let resolution = NamedSeeds::new(name_b.clone()) - .retry_interval(TEST_SEED_CADENCE) - .refresh_interval(TEST_SEED_CADENCE); + .with_retry_interval(TEST_SEED_CADENCE) + .with_refresh_interval(TEST_SEED_CADENCE); let node_a = WriteSync::bind(seed_config("seed-a", a, &[("seed-b", "b:1")]), resolution) .await .expect("binds loopback"); diff --git a/tests/it/common/faultnet.rs b/tests/it/common/faultnet.rs index ce5c602..435441f 100644 --- a/tests/it/common/faultnet.rs +++ b/tests/it/common/faultnet.rs @@ -139,6 +139,8 @@ impl Transport for FaultTransport { let inbound = Inbound { from: self.id.clone(), msg: msg.to_vec().into(), + // The fabric routes by identity and keeps no address book. + source: None, }; if delay.is_zero() { let _ = tx.send(inbound); diff --git a/tests/it/fleet_bootstrap.rs b/tests/it/fleet_bootstrap.rs index 26bbe1e..67b2818 100644 --- a/tests/it/fleet_bootstrap.rs +++ b/tests/it/fleet_bootstrap.rs @@ -2,6 +2,7 @@ use crate::common; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; @@ -718,9 +719,9 @@ async fn unreachable_donor_falls_back_to_origin_within_budget() { &broken_book, )) .with_recovery_config(RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 20_000, attempt_ms: 3_000, settle_ms: 3_000, @@ -1151,9 +1152,9 @@ const SLOW_READY_DEADLINE: Duration = Duration::from_secs(90); /// policy derived from it gives a builder a 4 s donor wait. fn stall_bounds() -> RecoveryConfig { RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 8_000, attempt_ms: 3_000, settle_ms: 3_000, @@ -1411,9 +1412,9 @@ async fn concurrent_start_follower_waits_out_a_production_paced_builder() { /// 30 s donor wait. fn short_episode() -> RecoveryConfig { RecoveryConfig { - max_members: 256, - max_member_bytes: 256, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(256).unwrap(), + max_member_bytes: NonZeroUsize::new(256).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 60_000, attempt_ms: 30_000, settle_ms: 3_000, diff --git a/tests/it/startup.rs b/tests/it/startup.rs index a5b0121..50ecb32 100644 --- a/tests/it/startup.rs +++ b/tests/it/startup.rs @@ -17,14 +17,15 @@ use s3cache::cache::proxy::CachingProxy; use s3cache::metrics::Metrics; use s3s::S3; use s3s::dto::GetObjectInput; +use std::num::{NonZeroU32, NonZeroUsize}; const CAP: usize = 1024 * 1024; fn short_recovery() -> RecoveryConfig { RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 5_000, attempt_ms: 250, settle_ms: 2_000, From 143ef43e73ea0d771a8ef0e34371f5f27e84959e Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:25:46 -0600 Subject: [PATCH 09/19] test(sync): write the malformed feed head under Groupnet's default feed key Groupnet now keys the default write feed's entry as ~writes/d, so a value written under the bare ~writes reached no observer and the observation stayed healthy. The test failed for that reason and passes with the real key. --- src/sync/volatile.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sync/volatile.rs b/src/sync/volatile.rs index 72ff44f..caf77a1 100644 --- a/src/sync/volatile.rs +++ b/src/sync/volatile.rs @@ -438,11 +438,13 @@ mod tests { poll_ms: 25, }; assert!(observe_group(&group_b, &sync, &b, Duration::from_millis(300), limits).is_ok()); + // Groupnet keys the default write feed's entry as `~writes/d`. + let feed_entry = "~writes/d"; group_a - .set_entry("~writes", vec![0xFF], None) + .set_entry(feed_entry, vec![0xFF], None) .expect("malformed entry enqueued"); tokio::time::timeout(Duration::from_secs(3), async { - while group_b.node_entry(&a, "~writes").is_none() { + while group_b.node_entry(&a, feed_entry).is_none() { tokio::task::yield_now().await; } }) From 7ba792a2263e652c122f77377f140837cded4c27 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:27:29 -0600 Subject: [PATCH 10/19] docs: name the single gossip cadence and the new sync submodules --- AGENTS.md | 4 +++- README.md | 2 +- scripts/resilience_e2e.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fc4bf97..7d593e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,10 +53,12 @@ src/ sync/ mod.rs # sync link point: module docs and declarations only coherence.rs # gossip write-feed and coherence core - config.rs # gossip env/config parsing and node construction + coherence/apply.rs # the apply loop: peer writes into the index, acks, gaps and lapses + config.rs # gossip env/config parsing and managed node construction volatile.rs # S3 origin facts/effects for Groupnet's bounded recovery driver wire.rs # write-event and session-token codec tests.rs # coherence unit tests + tests/fixtures.rs # managed in-memory nodes and wired pairs for those tests metrics.rs # counters, warm-residency gauges, and periodic stats logging tests/it/ # one integration-test binary; each module is a suite (links the library crate) deploy/helm/s3cache/ # Helm chart diff --git a/README.md b/README.md index 11a393e..588bcde 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ heuristic earlier releases used: after the lapse (so every wait held against the old one is resolved — read per granter rather than off the confirmed watermark, which is a *min* over the roster and advances when the member pinning it merely departs), settles for the fabric's - entry-propagation bound (`2 × anti_entropy_interval`), checks that no peer which was + entry-propagation bound (`2 × gossip_interval`), checks that no peer which was live at the lapse has vanished from membership since, and then barriers on every peer's advertised feed head — re-running the vanished check once more afterwards, because the barrier waits and a reap can land inside that wait. Past the barrier the apply loop has diff --git a/scripts/resilience_e2e.py b/scripts/resilience_e2e.py index ec806a8..eb070f5 100755 --- a/scripts/resilience_e2e.py +++ b/scripts/resilience_e2e.py @@ -71,7 +71,7 @@ FAST_MS = 500 # How long the returning node gets to settle before anything is asserted about it: its -# lease warm-up (one detection window plus two anti-entropy rounds) and its bootstrap LIST. +# lease warm-up (one detection window plus two gossip rounds) and its bootstrap LIST. # Below that it is correct but origin-served, which is exactly what the polls tolerate. REJOIN_SETTLE_S = LEASE_MS / 1000.0 + 1.0 From 6b577d6db48fc8caa8cd0141b373caaa81faa240 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:34:11 -0600 Subject: [PATCH 11/19] refactor(sync): keep RecoveryConfigRejected in sync::config, not the oversized proxy module --- src/cache/proxy.rs | 40 +--------------------------------------- src/sync/config.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/cache/proxy.rs b/src/cache/proxy.rs index 21eeb21..92cf325 100644 --- a/src/cache/proxy.rs +++ b/src/cache/proxy.rs @@ -20,6 +20,7 @@ use crate::index::{ }; use crate::metrics::Metrics; use crate::sync::coherence::{READ_TOKEN_HEADER, WRITE_TOKEN_HEADER, WriteReceipt, WriteSync}; +use crate::sync::config::RecoveryConfigRejected; use crate::sync::wire::wire_stamp; use crate::tier::{self, CachedObject, TieredCache, WarmPair}; use http::{HeaderMap, HeaderName, HeaderValue}; @@ -414,45 +415,6 @@ impl IndexedWrite { } } -/// Why [`CachingProxy::with_recovery_config`] refused a recovery configuration. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RecoveryConfigRejected { - /// Groupnet's own bounds refused it. - Invalid(groupnet::core::volatile_recovery::RecoveryConfigError), - /// The settle window is shorter than two gossip rounds of this node, so a - /// renewal could still be in flight when the window closes. - SettleBelowGossip { - /// The configured settle window. - settle_ms: u64, - /// Two of this node's gossip rounds. - floor_ms: u64, - }, -} - -impl std::fmt::Display for RecoveryConfigRejected { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Invalid(error) => error.fmt(f), - Self::SettleBelowGossip { - settle_ms, - floor_ms, - } => write!( - f, - "recovery settle_ms {settle_ms} is below two gossip rounds ({floor_ms} ms)" - ), - } - } -} - -impl std::error::Error for RecoveryConfigRejected { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Invalid(error) => Some(error), - Self::SettleBelowGossip { .. } => None, - } - } -} - /// S3 service that caches LIST (from an in-memory index) and small GET/HEAD bodies (the /// hot/warm/cold [`TieredCache`]) in front of an upstream `s3s_aws::Proxy`, forwarding /// every write. diff --git a/src/sync/config.rs b/src/sync/config.rs index 01ea34d..3d67171 100644 --- a/src/sync/config.rs +++ b/src/sync/config.rs @@ -284,3 +284,43 @@ pub(super) fn parse_seeds(raw: &str) -> Vec<(String, String)> { }) .collect() } + +/// Why [`CachingProxy::with_recovery_config`](crate::cache::proxy::CachingProxy::with_recovery_config) +/// refused a recovery configuration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecoveryConfigRejected { + /// Groupnet's own bounds refused it. + Invalid(groupnet::core::volatile_recovery::RecoveryConfigError), + /// The settle window is shorter than two gossip rounds of this node, so a + /// renewal could still be in flight when the window closes. + SettleBelowGossip { + /// The configured settle window. + settle_ms: u64, + /// Two of this node's gossip rounds. + floor_ms: u64, + }, +} + +impl std::fmt::Display for RecoveryConfigRejected { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Invalid(error) => error.fmt(f), + Self::SettleBelowGossip { + settle_ms, + floor_ms, + } => write!( + f, + "recovery settle_ms {settle_ms} is below two gossip rounds ({floor_ms} ms)" + ), + } + } +} + +impl std::error::Error for RecoveryConfigRejected { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Invalid(error) => Some(error), + Self::SettleBelowGossip { .. } => None, + } + } +} From 094399249f5bd805e55e2f16334144622e1be5e4 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 03:36:38 -0600 Subject: [PATCH 12/19] ci: run the test gate on pull requests and run every unit test The single test step passed integration-module filters that also applied to the lib and bin harnesses, so CI ran none of the 182 unit tests. Unit tests now run in their own step. Pull requests run the test job only; the image push and the Fleet bump stay on pushes to main. The MinIO-backed integration suites still run locally only. --- .github/workflows/build.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae34d32..5090a23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,18 @@ on: - ".dockerignore" - "deploy/**" - ".github/workflows/build.yml" + # Pull requests run the test gate only; the image push and the Fleet bump stay on main. + pull_request: + paths: + - "src/**" + - "tests/**" + - "scripts/**" + - "Cargo.toml" + - "Cargo.lock" + - "Dockerfile" + - ".dockerignore" + - "deploy/**" + - ".github/workflows/build.yml" workflow_dispatch: permissions: @@ -30,7 +42,8 @@ permissions: jobs: # Locked Rust lint/test/docs gate plus Helm lint/render. Run the real MinIO tests - # locally. CI runs unit tests and the integration modules that do not need an origin. + # locally. CI runs every unit test and the integration modules that do not need an + # origin. The unit tests run in their own step: a test-name filter would skip them. test: runs-on: ubuntu-latest permissions: @@ -44,7 +57,8 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all --check - run: cargo clippy --locked --workspace --all-targets --all-features - - run: "cargo test --locked --workspace --all-features --lib --bins --test it -- metrics_endpoint:: tier_cache:: startup_readiness::" + - run: cargo test --locked --workspace --all-features --lib --bins + - run: "cargo test --locked --workspace --all-features --test it -- metrics_endpoint:: tier_cache:: startup_readiness::" - run: cargo doc --locked --workspace --all-features --no-deps env: RUSTDOCFLAGS: -D warnings @@ -68,6 +82,7 @@ jobs: # reuse across builds. provenance/sbom off: the attestation stages turn the export # into a manifest list and break registry cache reuse (build-push-action#820). server: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read @@ -104,6 +119,7 @@ jobs: # The tag never moves backwards: an older run that finishes after a newer one keeps # the newer tag. bump: + if: github.event_name != 'pull_request' needs: [test, server] runs-on: ubuntu-latest permissions: From 4427328939fe41b2e91cc1d43f0fcf922695dbf6 Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:03:01 -0600 Subject: [PATCH 13/19] feat(sync): fence strong writes on every peer before the origin call --- src/cache/copy.rs | 5 +- src/cache/mutation.rs | 135 +++++++++++--- src/cache/proxy.rs | 73 +++++--- src/cache/stop.rs | 2 +- src/cache/tests.rs | 2 +- src/index.rs | 2 + src/index/fleet.rs | 3 + src/index/fleet/install.rs | 1 + src/index/fleet/snapshot.rs | 12 +- src/index/pending.rs | 352 ++++++++++++++++++++++++++++++++++++ src/sync/coherence.rs | 86 +++++++-- src/sync/coherence/apply.rs | 34 +++- src/sync/coherence/fence.rs | 169 +++++++++++++++++ src/sync/tests.rs | 9 +- src/sync/wire.rs | 28 ++- 15 files changed, 820 insertions(+), 93 deletions(-) create mode 100644 src/index/pending.rs create mode 100644 src/sync/coherence/fence.rs diff --git a/src/cache/copy.rs b/src/cache/copy.rs index b5ea6c8..dc75fb5 100644 --- a/src/cache/copy.rs +++ b/src/cache/copy.rs @@ -117,8 +117,11 @@ pub(super) fn indexed_source_size( return None; }; let state = proxy.state.read().unwrap(); + if state.key_fenced(bucket.as_ref(), key.as_ref()) { + return None; + } let bucket = state.get(bucket.as_ref())?; - if !bucket.synced || bucket.uncertain_keys.contains_key(key.as_ref()) { + if !bucket.synced { return None; } let entry = bucket.keys.get(key.as_ref())?; diff --git a/src/cache/mutation.rs b/src/cache/mutation.rs index bf60c7a..e4d9816 100644 --- a/src/cache/mutation.rs +++ b/src/cache/mutation.rs @@ -3,11 +3,14 @@ //! //! Each mutation's origin call and coherence tail run on a task of their own //! ([`CachingProxy::mutation_tail`]), so a client that hangs up cannot strand an applied -//! write outside the index. A success indexes and advertises what was written and holds -//! the response until the cluster has applied it. A failure that does not prove the -//! origin refused the write leaves the key's state unknown: it is fenced on every node -//! and reconciled from the origin ([`CachingProxy::publish_unknown`]) before the error -//! is answered, so no node answers that key from local state in the meantime. +//! write outside the index. In `strong`, the write is first announced to every peer +//! ([`WriteSync::fence`](crate::sync::coherence::WriteSync::fence)), so a writer that +//! dies after the origin applied it leaves no peer answering the key from local state. +//! A success indexes and advertises what was written and holds the response until the +//! cluster has applied it. A failure that does not prove the origin refused the write +//! leaves the key's state unknown: it is fenced on every node and reconciled from the +//! origin ([`CachingProxy::publish_unknown`]) before the error is answered. A refusal +//! abandons the announcement. use std::collections::BTreeSet; use std::sync::Arc; @@ -22,9 +25,11 @@ use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; use crate::cache::copy; use crate::cache::proxy::{ - CachingProxy, IndexedWrite, observed_entry, write_storage_class, written_object, + CachingProxy, IndexedWrite, WRITE_ACK_TIMEOUT, observed_entry, write_storage_class, + written_object, }; use crate::index::{ObjEntry, ObjMeta}; +use crate::sync::coherence::PendingWrite; use crate::tier::CachedObject; /// Why a failed mutation cannot be assumed not to have reached durable origin state. A @@ -59,6 +64,58 @@ fn copy_conflict_needs_reconcile(error: &S3Error, create_only: bool) -> bool { } impl CachingProxy { + /// Announce a write of `keys` to every peer before it reaches the origin. A stalled + /// announcement is a 503, and nothing reached the origin. + async fn fence_writes( + &self, + bucket: &str, + keys: &[String], + ) -> S3Result>> { + let Some(sync) = self.sync.as_ref() else { + return Ok(keys.iter().map(|_| None).collect()); + }; + sync.fence(&self.state, bucket, keys, WRITE_ACK_TIMEOUT, &self.metrics) + .await + .map_err(|_| { + s3s::s3_error!( + ServiceUnavailable, + "a cache peer could not be told about the write in time; it was not \ + sent to the origin, retry it" + ) + }) + } + + async fn fence_write(&self, bucket: &str, key: &str) -> S3Result> { + let mut pending = self.fence_writes(bucket, &[key.to_owned()]).await?; + Ok(pending.pop().flatten()) + } + + /// Settle a write the origin answered with an error: fenced and reconciled when the + /// error does not prove the origin refused it (`uncertain`), abandoned when it does. + async fn settle_failed( + &self, + bucket: &str, + key: &str, + uncertain: Option<&str>, + pending: Option, + ) { + match uncertain { + Some(reason) => { + self.publish_unknown(bucket, key, reason, pending.as_ref()) + .await; + } + None => self.abandon(bucket, key, pending).await, + } + } + + /// Settle a pending write that left the key as it was. + async fn abandon(&self, bucket: &str, key: &str, pending: Option) { + if let (Some(sync), Some(pending)) = (self.sync.as_ref(), pending) { + sync.index_abandoned(&self.state, bucket, key, pending, &self.metrics) + .await; + } + } + /// Forward a `PutObject`, then update the index from the result — and, when the write /// knows exactly what a read of it will report, keep the body it just wrote rather /// than dropping it (see [`buffered_put_body`](CachingProxy::buffered_put_body)). @@ -122,12 +179,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.put_object(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_mutation_error(&error, locally_vouched) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_mutation_error(&error, locally_vouched); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -149,7 +208,7 @@ impl CachingProxy { worker.metrics.write_fill(); } let token = worker - .record_put(IndexedWrite::Put, &bucket, &key, entry) + .record_put(IndexedWrite::Put, &bucket, &key, entry, pending.as_ref()) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) @@ -168,12 +227,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.delete_object(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_mutation_error(&error, false) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_mutation_error(&error, false); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -187,10 +248,10 @@ impl CachingProxy { // from the origin until an origin HEAD says. let receipt = if versioned { worker - .announce_unknown(&bucket, &key, "a version-scoped delete") + .announce_unknown(&bucket, &key, "a version-scoped delete", pending.as_ref()) .await } else { - worker.record_del(&bucket, &key).await + worker.record_del(&bucket, &key, pending.as_ref()).await }; let token = worker.await_cluster(receipt, &bucket, &key).await?; Self::attach_token(&mut resp.headers, token); @@ -216,21 +277,27 @@ impl CachingProxy { let keys: Vec = requested.iter().map(|(key, _)| key.clone()).collect(); let worker = self.clone(); let tail_bucket = bucket.clone(); + let fenced = keys.clone(); self.mutation_tail(&tail_bucket, &keys, async move { + let pending = worker.fence_writes(&bucket, &fenced).await?; let mut resp = match worker.inner.delete_objects(req).await { Ok(resp) => resp, Err(error) => { if let Some(reason) = uncertain_mutation_error(&error, false) { let mut receipt = None; - for (key, _) in &requested { + for ((key, _), pending) in requested.iter().zip(pending) { receipt = worker - .announce_unknown(&bucket, key, reason) + .announce_unknown(&bucket, key, reason, pending.as_ref()) .await .or(receipt); } worker .settle_cluster(receipt, &bucket, "") .await; + } else { + for ((key, _), pending) in requested.iter().zip(pending) { + worker.abandon(&bucket, key, pending).await; + } } return Err(error); } @@ -256,13 +323,14 @@ impl CachingProxy { .filter_map(|d| d.key.as_deref()) .collect(); let mut receipt = None; - for (key, versioned) in &requested { + for ((key, versioned), pending) in requested.iter().zip(pending) { let applied = if quiet { !refused.contains(key.as_str()) } else { deleted.contains(key.as_str()) }; if !applied { + worker.abandon(&bucket, key, pending).await; continue; } worker @@ -275,10 +343,10 @@ impl CachingProxy { // version is not the key (see `delete`): its key's state is unknown. let published = if *versioned { worker - .announce_unknown(&bucket, key, "a version-scoped delete") + .announce_unknown(&bucket, key, "a version-scoped delete", pending.as_ref()) .await } else { - worker.record_del(&bucket, key).await + worker.record_del(&bucket, key, pending.as_ref()).await }; receipt = published.or(receipt); } @@ -301,12 +369,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.complete_multipart_upload(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_completion_error(&error) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_completion_error(&error); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -327,7 +397,13 @@ impl CachingProxy { let mut entry = observed_entry(observed.as_ref()); entry.etag = completed.or(entry.etag); let token = worker - .record_put(IndexedWrite::MultipartComplete, &bucket, &key, entry) + .record_put( + IndexedWrite::MultipartComplete, + &bucket, + &key, + entry, + pending.as_ref(), + ) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) @@ -353,10 +429,12 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match copy::forward(&worker.copy_inner, req).await { Ok(resp) => resp, Err(error) => { if copy_conflict_needs_reconcile(&error, create_only) { + worker.abandon(&bucket, &key, pending).await; // The origin's 412 proves an immutable create-only // destination already exists. That can race ahead of this // proxy's LIST index (for example after an interrupted @@ -375,8 +453,11 @@ impl CachingProxy { } else { worker.metrics.copy_conflict_reconcile_miss(); } - } else if let Some(reason) = uncertain_mutation_error(&error, false) { - worker.publish_unknown(&bucket, &key, reason).await; + } else { + let uncertain = uncertain_mutation_error(&error, false); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; } return Err(error); } @@ -419,7 +500,7 @@ impl CachingProxy { }; entry.etag = copied_etag.or(entry.etag); let token = worker - .record_put(IndexedWrite::Copy, &bucket, &key, entry) + .record_put(IndexedWrite::Copy, &bucket, &key, entry, pending.as_ref()) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) diff --git a/src/cache/proxy.rs b/src/cache/proxy.rs index 92cf325..3c14200 100644 --- a/src/cache/proxy.rs +++ b/src/cache/proxy.rs @@ -19,7 +19,9 @@ use crate::index::{ uncertain_key_is_current, }; use crate::metrics::Metrics; -use crate::sync::coherence::{READ_TOKEN_HEADER, WRITE_TOKEN_HEADER, WriteReceipt, WriteSync}; +use crate::sync::coherence::{ + PendingWrite, READ_TOKEN_HEADER, WRITE_TOKEN_HEADER, WriteReceipt, WriteSync, +}; use crate::sync::config::RecoveryConfigRejected; use crate::sync::wire::wire_stamp; use crate::tier::{self, CachedObject, TieredCache, WarmPair}; @@ -86,7 +88,7 @@ impl CachedRange { /// uniform leased fleet): the lease wait runs to its own `D + 1s` deadline, and the two /// run concurrently, so the response is held for the longer of the two — see /// [`WriteSync::wait_cluster_applied`](crate::sync::coherence::WriteSync). -const WRITE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +pub(super) const WRITE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); /// How long a strict read waits for the freshness barrier before failing closed to the /// origin. A healthy cluster resolves this immediately. @@ -742,7 +744,7 @@ impl CachingProxy { let Some(bucket) = g.get(input.bucket.as_str()).filter(|bucket| bucket.synced) else { return false; }; - if bucket.uncertain_keys.contains_key(input.key.as_str()) { + if g.key_fenced(&input.bucket, &input.key) { return false; } let entry = bucket.keys.get(input.key.as_str()); @@ -786,9 +788,15 @@ impl CachingProxy { /// until the cluster has fenced it too or lost the right to serve, exactly as a /// describable write is held ([`record_put`](Self::record_put)). A stalled wait /// changes nothing here: the caller's error already tells the client the outcome is - /// unknown. - pub(super) async fn publish_unknown(&self, bucket: &str, key: &str, reason: &str) { - let receipt = self.announce_unknown(bucket, key, reason).await; + /// unknown. `pending` is the write's own announcement, which this settles. + pub(super) async fn publish_unknown( + &self, + bucket: &str, + key: &str, + reason: &str, + pending: Option<&PendingWrite>, + ) { + let receipt = self.announce_unknown(bucket, key, reason, pending).await; self.settle_cluster(receipt, bucket, key).await; } @@ -800,6 +808,7 @@ impl CachingProxy { bucket: &str, key: &str, reason: &str, + pending: Option<&PendingWrite>, ) -> Option { let ckey = (bucket.to_owned(), key.to_owned()); let Some(sync) = self.sync.as_ref() else { @@ -807,7 +816,8 @@ impl CachingProxy { self.obj_cache.invalidate(&ckey).await; return None; }; - let (token, publishing) = sync.index_unknown(&self.state, bucket, key, &self.metrics); + let (token, publishing) = + sync.index_unknown(&self.state, bucket, key, pending, &self.metrics); spawn_key_reconcile(&self.client, &self.state, bucket, key, token, reason); // Fenced first, so no fill can commit the old body behind this: a copy left in // place would be served again, trusted, once the reconciliation resolves. @@ -815,14 +825,10 @@ impl CachingProxy { Some(publishing.await) } - /// A key fenced by an ambiguous mutation may only be read from the origin until its - /// exact reconciliation token is resolved or superseded. - fn key_uncertain(&self, bucket: &str, key: &str) -> bool { - self.state - .read() - .unwrap() - .get(bucket) - .is_some_and(|bucket| bucket.uncertain_keys.contains_key(key)) + /// A key fenced by an ambiguous mutation, or by a peer's pending write, may only be + /// read from the origin until that fence is resolved or settled. + fn key_fenced(&self, bucket: &str, key: &str) -> bool { + self.state.read().unwrap().key_fenced(bucket, key) } /// Attach a write's session token to a response, when coherence is on. @@ -848,20 +854,26 @@ impl CachingProxy { /// coherence is on, or a 503 when the wait stalled (see /// [`await_cluster`](Self::await_cluster)). The caller has already dropped this /// node's own body copy — the writer must not serve its own stale bytes while the - /// cluster round runs. + /// cluster round runs. `pending` is the write's own announcement, which this settles. pub(super) async fn record_put( &self, op: IndexedWrite, bucket: &str, key: &str, mut entry: ObjEntry, + pending: Option<&PendingWrite>, ) -> S3Result> { // A write whose response carried neither an `ETag` nor a size describes nothing // a reader could check a copy against: it is reconciled as an uncertain one. if entry.etag.is_none() && entry.size.is_none() { op.record(&self.metrics); let receipt = self - .announce_unknown(bucket, key, "the write's response described no object") + .announce_unknown( + bucket, + key, + "the write's response described no object", + pending, + ) .await; return self.await_cluster(receipt, bucket, key).await; } @@ -875,7 +887,8 @@ impl CachingProxy { } return Ok(None); }; - let (changed, publishing) = sync.index_put(&self.state, bucket, key, entry, &self.metrics); + let (changed, publishing) = + sync.index_put(&self.state, bucket, key, entry, pending, &self.metrics); if changed { op.record(&self.metrics); } @@ -886,15 +899,21 @@ impl CachingProxy { /// Record a durable delete: tombstone + remove locally and advertise it to peers, /// returning the receipt rather than waiting. The caller chooses when to wait — one /// key waits immediately, a batch publishes every key first and then waits once (see - /// [`await_cluster`](Self::await_cluster)). - pub(super) async fn record_del(&self, bucket: &str, key: &str) -> Option { + /// [`await_cluster`](Self::await_cluster)). `pending` is the write's own + /// announcement, which this settles. + pub(super) async fn record_del( + &self, + bucket: &str, + key: &str, + pending: Option<&PendingWrite>, + ) -> Option { let ts = wire_stamp(SystemTime::now()); let Some(sync) = self.sync.as_ref() else { apply_del(&self.state, bucket, key, ts); return None; }; Some( - sync.index_del(&self.state, bucket, key, ts, &self.metrics) + sync.index_del(&self.state, bucket, key, ts, pending, &self.metrics) .await, ) } @@ -1219,7 +1238,7 @@ impl CachingProxy { || { self.obj_cache.suspect_gen() == generation && self.sync.as_ref().is_none_or(|sync| sync.may_serve_local()) - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) }, |obj, observed| { self.observe(&ckey.0, &ckey.1, observed); @@ -1275,9 +1294,7 @@ impl CachingProxy { resume_after: Option<&str>, ) -> Option { let g = self.state.read().unwrap(); - if g.get(inp.bucket.as_str()) - .is_some_and(|bucket| !bucket.uncertain_keys.is_empty()) - { + if g.fenced_under(&inp.bucket, inp.prefix.as_deref().unwrap_or_default()) { return None; } list_objects_v2_from_index( @@ -1412,7 +1429,7 @@ impl CachingProxy { // overwrite is never read stale; a token-carrying read that cannot be // verified in time skips every local copy and streams from the origin. let local_ok = if cacheable || cached_range.is_some() { - !self.key_uncertain(&ckey.0, &ckey.1) + !self.key_fenced(&ckey.0, &ckey.1) && self.read_barrier(&req.headers).await == ReadRoute::Local } else { true @@ -1518,7 +1535,7 @@ impl CachingProxy { let can_commit = || { self.obj_cache.suspect_gen() == generation && self.sync.as_ref().is_none_or(|sync| sync.may_serve_local()) - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) }; let on_commit = |filled: &Arc, observed: &ObservedObject| { // The GET supplies both body and HEAD metadata. Fold the observation @@ -1571,7 +1588,7 @@ impl CachingProxy { && req.input.if_unmodified_since.is_none(); let ckey = (req.input.bucket.clone(), req.input.key.clone()); if cache_eligible - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) && self.read_barrier(&req.headers).await == ReadRoute::Local { if let Some(obj) = self.validated_get(&ckey).await diff --git a/src/cache/stop.rs b/src/cache/stop.rs index cc88fe1..bb71372 100644 --- a/src/cache/stop.rs +++ b/src/cache/stop.rs @@ -140,7 +140,7 @@ impl CachingProxy { let mut receipt = None; for key in keys { receipt = self - .announce_unknown(bucket, key, "the mutation task terminated") + .announce_unknown(bucket, key, "the mutation task terminated", None) .await .or(receipt); } diff --git a/src/cache/tests.rs b/src/cache/tests.rs index ce0f2a7..806dc29 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -491,7 +491,7 @@ async fn a_stalled_write_is_a_503_not_a_success() { meta: None, }; let error = proxy - .record_put(crate::cache::proxy::IndexedWrite::Put, "bkt", "k", entry) + .record_put(crate::cache::proxy::IndexedWrite::Put, "bkt", "k", entry, None) .await .expect_err("a stalled write must not be acknowledged"); assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); diff --git a/src/index.rs b/src/index.rs index ed20218..b69863e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -29,6 +29,7 @@ use s3s::dto::{ }; pub(crate) mod fleet; +pub(crate) mod pending; mod rows; mod scan; use rows::{GoneRows, KeyRows}; @@ -250,6 +251,7 @@ struct KeyIndexState { stats: IndexStats, capture: Option, native_cuts: BTreeMap, (u64, u64)>, + pending: pending::PendingWrites, } /// Read guard over the key map. The aggregate statistics share its lock but remain an diff --git a/src/index/fleet.rs b/src/index/fleet.rs index 5cacf3d..3f90ca3 100644 --- a/src/index/fleet.rs +++ b/src/index/fleet.rs @@ -14,6 +14,7 @@ use groupnet::core::volatile_bootstrap::journal::{ use s3s::dto::{ETag, ObjectStorageClass}; +use super::pending::PendingWrites; use super::{BucketState, GoneRows, IndexStats, KeyIndexState, KeyRows, ObjEntry}; const VERSION: u8 = 1; @@ -955,6 +956,7 @@ pub(super) fn decode(bytes: &[u8], caps: ImageCaps) -> Result( index: &'a KeyIndexState, name: &'a str, @@ -26,7 +27,11 @@ fn image_bucket<'a>( } *previous = Some(name); let bucket = index.buckets.get(name).ok_or(ImageError::Incomplete)?; - if !bucket.synced || !bucket.uncertain_keys.is_empty() || bucket.rebuild_generation.is_some() { + if !bucket.synced + || !bucket.uncertain_keys.is_empty() + || index.pending.fenced_under(name, "") + || bucket.rebuild_generation.is_some() + { return Err(ImageError::Incomplete); } Ok(bucket) @@ -124,6 +129,7 @@ pub(crate) fn snapshot_state( stats: total, capture: None, native_cuts: BTreeMap::new(), + pending: PendingWrites::default(), }) } diff --git a/src/index/pending.rs b/src/index/pending.rs new file mode 100644 index 0000000..757b8ad --- /dev/null +++ b/src/index/pending.rs @@ -0,0 +1,352 @@ +//! Peer writes announced before their origin call and not settled yet. +//! +//! A `strong` writer publishes [`IndexOp::Pending`](crate::sync::wire::IndexOp) and +//! waits for its peers before it calls the origin. A reader holding one answers the +//! key from the origin until the writer settles it: a writer that crashes after the +//! origin applied its write never says what it did. A settling event names its pending +//! write, and every event carries the writer's oldest one still open, so a settle lost +//! to a gap is covered by the next event. What a writer's ended life left open is +//! fenced as uncertain instead (see [`fence_uncertain_key`](super::fence_uncertain_key)): +//! only the origin can say what it did. + +use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound; + +use groupnet::core::volatile_bootstrap::journal::{DeltaIdentity, NativeCut}; + +use super::{KeyIndex, KeyIndexReadGuard, KeyIndexState}; + +/// A key fenced as uncertain, with the token its origin reconciliation must present. +pub(crate) type UncertainKey = (String, String, u64); + +/// One writer life's open pending writes, by feed sequence. +struct Life { + epoch: u64, + open: BTreeMap, +} + +#[derive(Default)] +pub(super) struct PendingWrites { + writers: HashMap, Life>, + /// How many open pending writes fence each key, per bucket. + keys: HashMap>, +} + +impl PendingWrites { + #[cfg(test)] + fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + fn fenced(&self, bucket: &str, key: &str) -> bool { + self.keys + .get(bucket) + .is_some_and(|keys| keys.contains_key(key)) + } + + pub(super) fn fenced_under(&self, bucket: &str, prefix: &str) -> bool { + self.keys + .get(bucket) + .is_some_and(|keys| first_under(keys, prefix)) + } + + fn hold(&mut self, writer: &[u8], epoch: u64, seq: u64, bucket: &str, key: &str) { + let life = self.writers.entry(writer.to_vec()).or_insert_with(|| Life { + epoch, + open: BTreeMap::new(), + }); + if life.epoch != epoch { + return; + } + let entry = (bucket.to_owned(), key.to_owned()); + if life.open.insert(seq, entry).is_none() { + *self + .keys + .entry(bucket.to_owned()) + .or_default() + .entry(key.to_owned()) + .or_default() += 1; + } + } + + /// Settle `settles` and everything below `floor` (below `seq`, the settling + /// event's own position, when the writer had nothing open). + fn settle( + &mut self, + writer: &[u8], + epoch: u64, + seq: u64, + settles: Option, + floor: Option, + ) { + let Some(life) = self.writers.get_mut(writer) else { + return; + }; + if life.epoch != epoch { + return; + } + let mut done: Vec<(String, String)> = settles + .and_then(|settled| life.open.remove(&settled)) + .into_iter() + .collect(); + let still_open = life.open.split_off(&floor.unwrap_or(seq)); + done.extend(std::mem::replace(&mut life.open, still_open).into_values()); + if life.open.is_empty() { + self.writers.remove(writer); + } + for (bucket, key) in &done { + self.release(bucket, key); + } + } + + /// End `writer`'s life if it is older than `epoch`, returning what it left open. + fn end_life_before(&mut self, writer: &[u8], epoch: u64) -> Vec<(String, String)> { + if self + .writers + .get(writer) + .is_none_or(|life| life.epoch >= epoch) + { + return Vec::new(); + } + let Some(life) = self.writers.remove(writer) else { + return Vec::new(); + }; + let ended: Vec<(String, String)> = life.open.into_values().collect(); + for (bucket, key) in &ended { + self.release(bucket, key); + } + ended + } + + fn release(&mut self, bucket: &str, key: &str) { + let Some(keys) = self.keys.get_mut(bucket) else { + return; + }; + if let Some(count) = keys.get_mut(key) { + *count -= 1; + if *count == 0 { + keys.remove(key); + } + } + if keys.is_empty() { + self.keys.remove(bucket); + } + } +} + +/// Whether any key of a sorted set starts with `prefix`. +fn first_under(keys: &BTreeMap, prefix: &str) -> bool { + keys.range::((Bound::Included(prefix), Bound::Unbounded)) + .next() + .is_some_and(|(key, _)| key.starts_with(prefix)) +} + +impl KeyIndexState { + /// Fence what an ended writer life left pending as uncertain. + fn end_pending_life(&mut self, writer: &[u8], epoch: u64) -> Vec { + self.pending + .end_life_before(writer, epoch) + .into_iter() + .map(|(bucket, key)| { + let token = self.fence_key(&bucket, &key); + (bucket, key, token) + }) + .collect() + } +} + +impl KeyIndex { + /// Hold a peer's pending write under the lock that notes its feed position. The + /// event has no index effect, so the donor journal records a no-op for it. + /// Returns the keys an ended life of that writer left open, now uncertain. + pub(crate) fn hold_pending_native( + &self, + bucket: &str, + key: &str, + cut: &NativeCut, + floor: Option, + ) -> Vec { + let mut index = self.inner.write().unwrap(); + if !index.note_native_cut(cut) { + return Vec::new(); + } + if let Some(capture) = index.capture.as_mut() { + capture.record_native_noop(DeltaIdentity::Native(cut.clone())); + } + let ended = index.end_pending_life(&cut.writer, cut.epoch); + index + .pending + .settle(&cut.writer, cut.epoch, cut.sequence, None, floor); + index + .pending + .hold(&cut.writer, cut.epoch, cut.sequence, bucket, key); + ended + } + + /// Settle a peer's pending writes as a later event of its feed says, after that + /// event's own index effect. An [`IndexOp::Abandoned`](crate::sync::wire::IndexOp) + /// has none, so `noop` notes its position and journals it here. Returns the keys + /// an ended life of that writer left open, now uncertain. + pub(crate) fn settle_pending_native( + &self, + cut: &NativeCut, + settles: Option, + floor: Option, + noop: bool, + ) -> Vec { + let mut index = self.inner.write().unwrap(); + if noop { + if !index.note_native_cut(cut) { + return Vec::new(); + } + if let Some(capture) = index.capture.as_mut() { + capture.record_native_noop(DeltaIdentity::Native(cut.clone())); + } + } + let ended = index.end_pending_life(&cut.writer, cut.epoch); + index + .pending + .settle(&cut.writer, cut.epoch, cut.sequence, settles, floor); + ended + } + + /// End every pending write of `writer`'s lives before `epoch`: its feed moved + /// past them (a new life, or a seal). Returns those keys, now uncertain. + pub(crate) fn end_pending_life(&self, writer: &[u8], epoch: u64) -> Vec { + self.inner + .write() + .unwrap() + .end_pending_life(writer, epoch) + } + + /// Note this node's own feed event that changes no index row — its pending and + /// abandoned writes — under the lock that assigns its position, journaled as a + /// no-op so the donor journal stays contiguous. + pub(crate) fn note_own_native(&self, assign: impl FnOnce() -> (R, NativeCut)) -> R { + let mut index = self.inner.write().unwrap(); + let (value, position) = assign(); + if index.note_native_cut(&position) + && let Some(capture) = index.capture.as_mut() + { + capture.record_native_noop(DeltaIdentity::Native(position)); + } + value + } +} + +impl KeyIndexReadGuard<'_> { + /// Whether `key` may only be answered from the origin: its last mutation is + /// unreconciled, or a peer's write of it is pending. + pub(crate) fn key_fenced(&self, bucket: &str, key: &str) -> bool { + self.0.pending.fenced(bucket, key) + || self + .0 + .buckets + .get(bucket) + .is_some_and(|b| b.uncertain_keys.contains_key(key)) + } + + /// Whether any key under `prefix` is fenced (see [`key_fenced`](Self::key_fenced)). + pub(crate) fn fenced_under(&self, bucket: &str, prefix: &str) -> bool { + self.0.pending.fenced_under(bucket, prefix) + || self + .0 + .buckets + .get(bucket) + .is_some_and(|b| first_under(&b.uncertain_keys, prefix)) + } +} + +#[cfg(test)] +mod tests { + use groupnet::core::volatile_bootstrap::journal::NativeCut; + + use crate::index::{KeyIndex, uncertain_key_is_current}; + + fn cut(epoch: u64, sequence: u64) -> NativeCut { + NativeCut { + writer: b"writer".to_vec(), + epoch, + sequence, + } + } + + fn fenced(state: &KeyIndex, key: &str) -> bool { + state.read().unwrap().key_fenced("bkt", key) + } + + #[test] + fn a_settle_releases_only_the_pending_write_it_names() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "a", &cut(1, 1), None); + state.hold_pending_native("bkt", "b", &cut(1, 2), Some(1)); + state.settle_pending_native(&cut(1, 3), Some(2), Some(1), false); + assert!(fenced(&state, "a"), "a's write is still open"); + assert!(!fenced(&state, "b")); + state.settle_pending_native(&cut(1, 4), Some(1), None, false); + assert!(!fenced(&state, "a")); + assert!(state.read().unwrap().0.pending.is_empty()); + } + + /// Two writes of one key by one writer fence it until both settle. + #[test] + fn a_key_stays_fenced_while_any_write_of_it_is_open() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.hold_pending_native("bkt", "k", &cut(1, 2), Some(1)); + state.settle_pending_native(&cut(1, 3), Some(1), Some(2), false); + assert!(fenced(&state, "k")); + state.settle_pending_native(&cut(1, 4), Some(2), None, false); + assert!(!fenced(&state, "k")); + } + + /// The floor settles a write whose settling event a gap swallowed. + #[test] + fn the_floor_settles_a_write_whose_settle_was_lost() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "lost", &cut(1, 1), None); + state.hold_pending_native("bkt", "open", &cut(1, 2), Some(1)); + // Sequence 3 settled `lost` and never arrived; 4 says 2 is the oldest open. + state.settle_pending_native(&cut(1, 4), None, Some(2), true); + assert!(!fenced(&state, "lost")); + assert!(fenced(&state, "open")); + } + + /// A duplicate delivery changes nothing. + #[test] + fn a_duplicate_pending_event_is_held_once() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.settle_pending_native(&cut(1, 2), Some(1), None, true); + assert!(!fenced(&state, "k")); + } + + /// A writer's next life fences what its last one left open as uncertain: the + /// crashed life cannot say whether the origin applied it. + #[test] + fn a_new_life_turns_open_writes_into_uncertain_keys() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + assert!(state.end_pending_life(b"writer", 1).is_empty(), "same life"); + let ended = state.end_pending_life(b"writer", 2); + assert_eq!(ended.len(), 1); + let (bucket, key, token) = &ended[0]; + assert_eq!((bucket.as_str(), key.as_str()), ("bkt", "k")); + assert!(uncertain_key_is_current(&state, "bkt", "k", *token)); + assert!(state.read().unwrap().0.pending.is_empty()); + assert!(fenced(&state, "k"), "uncertain until the origin says"); + } + + #[test] + fn a_list_is_fenced_only_under_a_prefix_holding_a_fenced_key() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "log/0002", &cut(1, 1), None); + let guard = state.read().unwrap(); + assert!(guard.fenced_under("bkt", "")); + assert!(guard.fenced_under("bkt", "log/")); + assert!(!guard.fenced_under("bkt", "ptr/")); + assert!(!guard.fenced_under("bkt", "log/0003")); + assert!(!guard.fenced_under("other", "")); + } +} diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index 49c0d28..671ec82 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -1,4 +1,7 @@ mod apply; +mod fence; + +pub(crate) use fence::PendingWrite; use std::num::NonZeroUsize; use std::sync::{Arc, OnceLock}; @@ -17,6 +20,7 @@ use groupnet::runtime::{Group, Node}; use s3s::dto::ObjectStorageClass; use tracing::{info, warn}; +use crate::index::pending::UncertainKey; use crate::index::{ KeyIndex, ObjEntry, apply_del_native, apply_own_del, apply_own_put, apply_put_native, fence_own_uncertain_key, fence_uncertain_key_native, standard_class, @@ -45,20 +49,31 @@ pub(super) fn native_cut(peer: &NodeId, token: WriteToken) -> NativeCut { } /// Fold one peer event into the index. A put that describes nothing (see -/// [`IndexOp::unknown`]) fences its key instead, and the fence's token is returned for -/// the caller to reconcile with the origin. +/// [`IndexOp::unknown`]) fences its key instead; a pending write fences it until the +/// writer settles it. Returns every key left uncertain, for the caller to reconcile +/// with the origin. fn apply_feed_index( state: &KeyIndex, peer: &NodeId, token: WriteToken, event: &IndexEvent, -) -> Option { +) -> Vec { let ts = from_micros(event.ts_us); let cut = native_cut(peer, token); - if event.op.describes_nothing() { - return fence_uncertain_key_native(state, &event.bucket, &event.key, &cut); - } + let mut uncertain = Vec::new(); match &event.op { + IndexOp::Pending => { + return state.hold_pending_native(&event.bucket, &event.key, &cut, event.floor); + } + IndexOp::Abandoned => { + return state.settle_pending_native(&cut, event.settles, event.floor, true); + } + op if op.describes_nothing() => { + if let Some(fence) = fence_uncertain_key_native(state, &event.bucket, &event.key, &cut) + { + uncertain.push((event.bucket.clone(), event.key.clone(), fence)); + } + } IndexOp::Put { size, etag, @@ -77,13 +92,16 @@ fn apply_feed_index( content_type: content_type.clone(), meta: None, }; - apply_put_native(state, &event.bucket, &event.key, entry, cut); + apply_put_native(state, &event.bucket, &event.key, entry, cut.clone()); } IndexOp::Del => { - apply_del_native(state, &event.bucket, &event.key, ts, cut); + apply_del_native(state, &event.bucket, &event.key, ts, cut.clone()); } } - None + // The outcome is indexed first, so a read that no longer sees the pending write + // sees what replaced it. + uncertain.extend(state.settle_pending_native(&cut, event.settles, event.floor, false)); + uncertain } /// Starts an origin reconciliation of a key this node fenced under `token` (see @@ -367,6 +385,9 @@ pub struct WriteSync { /// The counted granters membership last did not report alive, as last logged /// (see [`absent_granters`](Self::absent_granters)). absent_granters: std::sync::Mutex>, + /// The feed sequences of this node's pending writes not settled yet (see + /// [`fence`](Self::fence)). + open_writes: fence::OpenWrites, } impl WriteSync { @@ -421,6 +442,7 @@ impl WriteSync { advertisement, unmet_seeds: Arc::default(), absent_granters: std::sync::Mutex::default(), + open_writes: fence::OpenWrites::default(), node, } } @@ -600,14 +622,16 @@ impl WriteSync { /// Index this node's own put and advertise it to peers: everything the entry /// carries that a peer can use, stamped with the local index's timestamp. The feed /// assigns the write's position under the index lock that applies it, so every own - /// write reaches the index, the donor journal and the feed in one order. Returns - /// whether the index changed and the advertisement to await. + /// write reaches the index, the donor journal and the feed in one order. `settles` + /// is the write's pending announcement, if it made one. Returns whether the index + /// changed and the advertisement to await. pub(crate) fn index_put<'a>( &'a self, state: &KeyIndex, bucket: &str, key: &str, entry: ObjEntry, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> (bool, impl Future + use<'a>) { let event = IndexEvent { @@ -620,9 +644,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(entry.last_modified), + settles: None, + floor: None, }; let (changed, publishing) = apply_own_put(state, bucket, key, entry, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); (changed, self.advertised(publishing, metrics)) } @@ -635,6 +661,7 @@ impl WriteSync { bucket: &str, key: &str, ts: SystemTime, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> impl Future + use<'a> { let event = IndexEvent { @@ -642,9 +669,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(ts), + settles: None, + floor: None, }; let (_, publishing) = apply_own_del(state, bucket, key, ts, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); self.advertised(publishing, metrics) } @@ -658,6 +687,7 @@ impl WriteSync { state: &KeyIndex, bucket: &str, key: &str, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> (u64, impl Future + use<'a>) { let event = IndexEvent { @@ -665,9 +695,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, }; let (token, publishing) = fence_own_uncertain_key(state, bucket, key, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); (token, self.advertised(publishing, metrics)) } @@ -822,6 +854,27 @@ impl WriteSync { bucket: &str, key: &str, metrics: &Metrics, + ) -> bool { + self.acked( + token, + timeout, + bucket, + key, + metrics, + "answering 503, the write may be applied at the origin", + ) + .await + } + + /// [`ack_write`](Self::ack_write), logging `on_stall` when the wait stalls. + async fn acked( + &self, + token: WriteToken, + timeout: Duration, + bucket: &str, + key: &str, + metrics: &Metrics, + on_stall: &str, ) -> bool { let outcome: WriteWait = self.wait_cluster_applied(token, timeout).await; let retires_feed = outcome.retires_feed(self.consistency); @@ -843,10 +896,7 @@ impl WriteSync { } WriteWait::Stalled { waiting_on, lapsed } => { let who = node_names(&waiting_on); - warn!( - "write ack timed out for {bucket}/{key} ({who}); \ - answering 503, the write may be applied at the origin" - ); + warn!("write ack timed out for {bucket}/{key} ({who}); {on_stall}"); metrics.ack_timeout(); if !lapsed.is_empty() { // The lease tier *did* resolve this write, on a lapse, and the diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs index 1d3eadc..d9b484d 100644 --- a/src/sync/coherence/apply.rs +++ b/src/sync/coherence/apply.rs @@ -6,6 +6,7 @@ use groupnet::core::NodeId; use tracing::warn; use crate::index::KeyIndex; +use crate::index::pending::UncertainKey; use crate::metrics::Metrics; use crate::sync::coherence::{KeyReconcile, WriteSync, apply_feed_index, lapse_poll, native_cut}; use crate::sync::wire::decode_event; @@ -65,6 +66,11 @@ impl WriteSync { match event { PeerWrite::Sealed { peer, token } => { state.note_native_seal(native_cut(&peer, token)); + let ended = state.end_pending_life( + peer.as_str().as_bytes(), + token.epoch.saturating_add(1), + ); + reconcile_all(&reconcile, ended); sync.observe_seal(&peer, token); frontier.advance(&peer, token); acknowledge(ledger.as_ref(), &peer, token).await; @@ -75,6 +81,10 @@ impl WriteSync { epoch, } => { let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); + reconcile_all( + &reconcile, + state.end_pending_life(peer.as_str().as_bytes(), epoch), + ); frontier.advance(&peer, start); acknowledge(ledger.as_ref(), &peer, start).await; } @@ -88,11 +98,9 @@ impl WriteSync { let fenced = apply_feed_index(&state, &peer, token, &event); drop(mutation); // Fenced before the acknowledgement below: from then on this - // node answers the key from the origin until the reconciliation - // resolves it. - if let Some(fence) = fenced { - reconcile(&event.bucket, &event.key, fence); - } + // node answers these keys from the origin until each + // reconciliation resolves its key. + reconcile_all(&reconcile, fenced); // The index must move first: a warm body promoted after this hot // eviction decodes suspect and is checked against the new entry // before it can be served. Awaiting moka removal before the ack @@ -116,6 +124,15 @@ impl WriteSync { warn!("write-feed gap from `{peer}`: distrusting bodies, resyncing index"); sync.forget_crossing(&peer); state.discard_for_gap(&native_cut(&peer, missed_through)); + // A gap within one life keeps that life's pending writes: + // the floor of its next event settles them. + reconcile_all( + &reconcile, + state.end_pending_life( + peer.as_str().as_bytes(), + missed_through.epoch, + ), + ); // The public signal fences serving before returning; the // recovery worker distrusts bodies and guards every origin // scan page with that exact operation's permit. Frontier @@ -135,6 +152,13 @@ impl WriteSync { } } +/// Start the origin reconciliation of each key left uncertain. +fn reconcile_all(reconcile: &KeyReconcile, keys: Vec) { + for (bucket, key, token) in keys { + reconcile(&bucket, &key, token); + } +} + /// Advertise that this node applied `peer`'s feed through `token`. A refused /// advertisement keeps the advance locally, and the next advancing record carries it. async fn acknowledge(ledger: Option<&AckLedger>, peer: &NodeId, token: WriteToken) { diff --git a/src/sync/coherence/fence.rs b/src/sync/coherence/fence.rs new file mode 100644 index 0000000..f096e64 --- /dev/null +++ b/src/sync/coherence/fence.rs @@ -0,0 +1,169 @@ +//! The pre-write fence. In `strong`, a write is announced to every peer, and the +//! announcement acknowledged (or the peer's lease lapsed), before the write reaches the +//! origin. A writer that crashes after the origin applied the write therefore leaves +//! every peer answering the key from the origin (see [`crate::index::pending`]). + +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, SystemTime}; + +use groupnet::consistency::{PublishError, WriteToken}; +use groupnet::core::volatile_bootstrap::journal::NativeCut; + +use crate::index::KeyIndex; +use crate::metrics::Metrics; +use crate::sync::coherence::{Consistency, WriteReceipt, WriteSync}; +use crate::sync::wire::{IndexEvent, IndexOp, to_micros}; + +type Open = Arc>>; + +fn lock(open: &Open) -> MutexGuard<'_, BTreeSet> { + open.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// The feed sequences of this node's pending writes not settled yet. +#[derive(Default)] +pub(super) struct OpenWrites(Open); + +/// A write announced to every peer before its origin call. The event reporting its +/// outcome settles it. Dropped unsettled (its task died), it stops holding back the +/// floor the writer's next event carries. +pub(crate) struct PendingWrite { + seq: u64, + open: Open, +} + +impl Drop for PendingWrite { + fn drop(&mut self) { + lock(&self.open).remove(&self.seq); + } +} + +/// The fence did not reach every peer in time. Its announcements are abandoned and +/// the write must not reach the origin. +#[derive(Debug)] +pub(crate) struct FenceStalled; + +impl WriteSync { + /// Stamp an own event with the pending write it settles and the oldest one still + /// open, and publish it. Runs under the index lock that orders every own event; + /// `opens` records the event as a new pending write. + pub(super) fn publish_own( + &self, + mut event: IndexEvent, + settles: Option<&PendingWrite>, + opens: bool, + ) -> ( + impl Future> + Send + use<'_>, + NativeCut, + ) { + let mut open = lock(&self.open_writes.0); + if let Some(pending) = settles { + open.remove(&pending.seq); + event.settles = Some(pending.seq); + } + event.floor = open.first().copied(); + let publishing = self.feed.publish(&event); + let position = self.own_position(); + if opens { + open.insert(position.sequence); + } + (publishing, position) + } + + /// Announce a write of each of `keys` and wait until every peer that could serve + /// them holds the announcement or has lapsed. Only then may the write reach the + /// origin, and each returned [`PendingWrite`] is settled by the event that reports + /// its key's outcome. Only `strong` fences; other modes get `None` for each key. + /// + /// # Errors + /// [`FenceStalled`] when the wait stalled. The announcements are then abandoned. + pub(crate) async fn fence( + &self, + state: &KeyIndex, + bucket: &str, + keys: &[String], + timeout: Duration, + metrics: &Metrics, + ) -> Result>, FenceStalled> { + if self.consistency != Consistency::Strong { + return Ok(keys.iter().map(|_| None).collect()); + } + let mut pending = Vec::with_capacity(keys.len()); + let mut newest = None; + for key in keys { + let (write, receipt) = self.index_pending(state, bucket, key, metrics).await; + pending.push(write); + newest = Some(receipt.token); + } + let Some(token) = newest else { + return Ok(Vec::new()); + }; + let label = match keys { + [key] => key.as_str(), + _ => "", + }; + let on_stall = "answering 503, the write was not sent to the origin"; + if self + .acked(token, timeout, bucket, label, metrics, on_stall) + .await + { + return Ok(pending.into_iter().map(Some).collect()); + } + for (key, write) in keys.iter().zip(pending) { + self.index_abandoned(state, bucket, key, write, metrics) + .await; + } + Err(FenceStalled) + } + + async fn index_pending( + &self, + state: &KeyIndex, + bucket: &str, + key: &str, + metrics: &Metrics, + ) -> (PendingWrite, WriteReceipt) { + let event = IndexEvent { + op: IndexOp::Pending, + bucket: bucket.to_owned(), + key: key.to_owned(), + ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, + }; + let (publishing, seq) = state.note_own_native(|| { + let (publishing, position) = self.publish_own(event, None, true); + ((publishing, position.sequence), position) + }); + let write = PendingWrite { + seq, + open: Arc::clone(&self.open_writes.0), + }; + (write, self.advertised(publishing, metrics).await) + } + + /// Settle a pending write the origin refused, or that was never sent: the key is + /// as it was, so peers stop fencing it without waiting for anyone. + pub(crate) async fn index_abandoned( + &self, + state: &KeyIndex, + bucket: &str, + key: &str, + write: PendingWrite, + metrics: &Metrics, + ) { + let event = IndexEvent { + op: IndexOp::Abandoned, + bucket: bucket.to_owned(), + key: key.to_owned(), + ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, + }; + let publishing = + state.note_own_native(|| self.publish_own(event, Some(&write), false)); + drop(write); + self.advertised(publishing, metrics).await; + } +} diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 5c0878d..94954d8 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -38,7 +38,7 @@ fn test_lease() -> LeaseConfig { /// Index `entry` as this writer's own put into a scratch index and advertise it. async fn own_put(sync: &WriteSync, key: &str, entry: ObjEntry, metrics: &Metrics) -> WriteReceipt { - sync.index_put(&KeyIndex::default(), "bkt", key, entry, metrics) + sync.index_put(&KeyIndex::default(), "bkt", key, entry, None, metrics) .1 .await } @@ -438,6 +438,8 @@ fn event_codec_round_trips_and_rejects_garbage() { bucket: "bucket-1".to_owned(), key: "a/b weird\0key".to_owned(), ts_us: 1_700_000_000_000_000, + settles: Some(7), + floor: Some(5), }; let encoded = encode_event(&event); assert_eq!( @@ -534,6 +536,7 @@ async fn peer_events_fold_into_index_and_invalidate() { "bkt", "obj", SystemTime::now(), + None, &metrics, ) .await; @@ -585,6 +588,7 @@ async fn peer_delete_retires_the_warm_copy() { "bkt", "gone", SystemTime::now(), + None, &metrics, ) .await; @@ -640,7 +644,8 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { let ckey = ("bkt".to_owned(), "ptr".to_owned()); cache.insert(ckey.clone(), cached(b"old")).await; - let (_, publishing) = sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", &metrics); + let (_, publishing) = + sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", None, &metrics); let unknown = publishing.await; assert!( sync_b diff --git a/src/sync/wire.rs b/src/sync/wire.rs index ad8ec66..f3b4c59 100644 --- a/src/sync/wire.rs +++ b/src/sync/wire.rs @@ -15,11 +15,12 @@ pub(super) fn parse_token(value: &str) -> Option<(&str, u64, u64)> { /// Prefix identifying the current event envelope: a postcard `IndexEvent`. Bytes /// without it are not this protocol and are rejected rather than interpreted as -/// another shape. Peers upgrade together, so the retired bincode envelope (`0xFF`) -/// is not read: mixed-version fleets are unsupported, as for every peer protocol. -pub(super) const WIRE_MAGIC: u8 = 0xFE; +/// another shape. Peers upgrade together, so the retired envelopes — bincode (`0xFF`) +/// and the postcard event before the pre-write fence (`0xFE`) — are not read: +/// mixed-version fleets are unsupported, as for every peer protocol. +pub(super) const WIRE_MAGIC: u8 = 0xFD; -/// What one durable write did, as advertised to peers. +/// What one write did, or is about to do, as advertised to peers. #[derive(Serialize, Deserialize)] pub(crate) enum IndexOp { /// The key now holds an object. @@ -41,6 +42,13 @@ pub(crate) enum IndexOp { }, /// The key was deleted. Del, + /// A `strong` write of the key is about to reach the origin. Until the writer + /// settles it, a reader answers the key from the origin: a writer that crashes + /// after the origin applied the write cannot tell anyone what it did. + Pending, + /// The origin refused the pending write, or it was never sent: the key is as + /// it was. It only settles the pending write it names. + Abandoned, } impl IndexOp { @@ -75,15 +83,21 @@ impl IndexOp { } } -/// One durable write: the operation, its `(bucket, key)`, and the writer's wall-clock -/// timestamp — the cross-writer LWW tiebreak, in **microseconds** so it is not coarser -/// than the clock a local write is stamped with (see [`wire_stamp`]). +/// One write-feed event: the operation, its `(bucket, key)`, and the writer's +/// wall-clock timestamp — the cross-writer LWW tiebreak, in **microseconds** so it is +/// not coarser than the clock a local write is stamped with (see [`wire_stamp`]). #[derive(Serialize, Deserialize)] pub(crate) struct IndexEvent { pub(crate) op: IndexOp, pub(crate) bucket: String, pub(crate) key: String, pub(crate) ts_us: u64, + /// The feed sequence of the [`IndexOp::Pending`] this event settles. + pub(crate) settles: Option, + /// The writer's oldest pending write still open when this event was published; + /// `None` when none was. A reader settles every older one it holds, which also + /// covers a settling event it lost to a gap. + pub(crate) floor: Option, } pub(crate) fn to_micros(ts: SystemTime) -> u64 { From 1039bec7084142330f81bde912fdb2a813ecd67e Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:08:26 -0600 Subject: [PATCH 14/19] style(sync): rustfmt the pre-write fence --- src/index/pending.rs | 5 +---- src/sync/coherence/apply.rs | 5 +---- src/sync/coherence/fence.rs | 3 +-- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/index/pending.rs b/src/index/pending.rs index 757b8ad..0e626da 100644 --- a/src/index/pending.rs +++ b/src/index/pending.rs @@ -213,10 +213,7 @@ impl KeyIndex { /// End every pending write of `writer`'s lives before `epoch`: its feed moved /// past them (a new life, or a seal). Returns those keys, now uncertain. pub(crate) fn end_pending_life(&self, writer: &[u8], epoch: u64) -> Vec { - self.inner - .write() - .unwrap() - .end_pending_life(writer, epoch) + self.inner.write().unwrap().end_pending_life(writer, epoch) } /// Note this node's own feed event that changes no index row — its pending and diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs index d9b484d..70639e6 100644 --- a/src/sync/coherence/apply.rs +++ b/src/sync/coherence/apply.rs @@ -128,10 +128,7 @@ impl WriteSync { // the floor of its next event settles them. reconcile_all( &reconcile, - state.end_pending_life( - peer.as_str().as_bytes(), - missed_through.epoch, - ), + state.end_pending_life(peer.as_str().as_bytes(), missed_through.epoch), ); // The public signal fences serving before returning; the // recovery worker distrusts bodies and guards every origin diff --git a/src/sync/coherence/fence.rs b/src/sync/coherence/fence.rs index f096e64..1f8915f 100644 --- a/src/sync/coherence/fence.rs +++ b/src/sync/coherence/fence.rs @@ -161,8 +161,7 @@ impl WriteSync { settles: None, floor: None, }; - let publishing = - state.note_own_native(|| self.publish_own(event, Some(&write), false)); + let publishing = state.note_own_native(|| self.publish_own(event, Some(&write), false)); drop(write); self.advertised(publishing, metrics).await; } From 5418463df73992ac54ff9caf45d01c73fc9e879a Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:08:26 -0600 Subject: [PATCH 15/19] test(sync): cover the pre-write fence's crash points, stall and lapse --- src/cache/tests.rs | 105 +++++++++++++++++++++++- src/sync/tests.rs | 196 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 6 deletions(-) diff --git a/src/cache/tests.rs b/src/cache/tests.rs index 806dc29..406fabb 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -1,5 +1,6 @@ use std::num::NonZeroUsize; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, SystemTime}; use bytes::Bytes; @@ -11,7 +12,9 @@ use groupnet::runtime::Node; use groupnet::transport::link::{BoundLink, LinkConfig}; use groupnet::transport::mem::Network; use http::HeaderMap; -use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, ListObjectsV2Input, Timestamp}; +use s3s::dto::{ + ETag, GetObjectInput, GetObjectOutput, ListObjectsV2Input, PutObjectInput, Timestamp, +}; use s3s::{S3, S3ErrorCode, S3Request}; use crate::cache::proxy::{CacheConfig, CachingProxy, FullSyncOwner, ObservedObject, ReadRoute}; @@ -76,6 +79,11 @@ fn only_the_newest_single_node_warmup_may_retry() { /// A proxy over an origin it never dials. Every case below is decided from local /// state; a row that reached the endpoint would fail on the connection, not pass. fn proxy(sync: Option>) -> CachingProxy { + proxy_at("http://127.0.0.1:1", sync) +} + +/// A proxy whose origin is `endpoint`. +fn proxy_at(endpoint: &str, sync: Option>) -> CachingProxy { let conf = aws_sdk_s3::config::Builder::new() .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) .region(aws_sdk_s3::config::Region::new("us-east-1")) @@ -86,7 +94,7 @@ fn proxy(sync: Option>) -> CachingProxy { None, "s3cache-unit-tests", )) - .endpoint_url("http://127.0.0.1:1") + .endpoint_url(endpoint) .force_path_style(true) .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) .build(); @@ -491,13 +499,104 @@ async fn a_stalled_write_is_a_503_not_a_success() { meta: None, }; let error = proxy - .record_put(crate::cache::proxy::IndexedWrite::Put, "bkt", "k", entry, None) + .record_put( + crate::cache::proxy::IndexedWrite::Put, + "bkt", + "k", + entry, + None, + ) .await .expect_err("a stalled write must not be acknowledged"); assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); assert_eq!(counter(&proxy, "ack_timeouts"), 1); } +async fn linked(net: &Network, id: &NodeId, peer: &NodeId) -> Node { + Node::builder(id.clone()) + .link(BoundLink::new( + net.endpoint(id.clone()), + LinkConfig::new(vec![peer.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) + .config(brisk()) + .start() + .await + .expect("managed node starts") +} + +/// The fence comes before the origin call. A write whose fence cannot complete is a 503 +/// that never reached the origin, which is also all a writer that dies before its fence +/// completes can have done. The announcement is abandoned, not left open. +#[tokio::test] +async fn a_stalled_fence_is_a_503_that_never_reaches_the_origin() { + let origin = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("the origin binds"); + let endpoint = format!("http://{}", origin.local_addr().expect("origin address")); + let dialed = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&dialed); + tokio::spawn(async move { + while origin.accept().await.is_ok() { + counted.fetch_add(1, Ordering::SeqCst); + } + }); + let net = Network::new(); + let (a_id, b_id) = (NodeId::new("fence-stall-a"), NodeId::new("fence-stall-b")); + let a_node = linked(&net, &a_id, &b_id).await; + let b_node = linked(&net, &b_id, &a_id).await; + let lease = LeaseConfig::for_duration(Duration::from_millis(300)); + // A is in the ack tier without a lease and runs no apply loop: B's fence waits on + // it and never hears back. + let _peer = WriteSync::attach( + a_node.join_group("s3cache"), + a_id.clone(), + Consistency::StrongAcks, + lease, + None, + ); + let b_group = b_node.join_group("s3cache"); + let sync = Arc::new(WriteSync::attach( + b_group.clone(), + b_id, + Consistency::Strong, + lease, + None, + )); + let proxy = proxy_at(&endpoint, Some(sync)); + tokio::time::timeout(Duration::from_secs(2), async { + while !b_group.members().contains(&a_id) { + tokio::task::yield_now().await; + } + }) + .await + .expect("the peers converge"); + + let error = proxy + .put_object(request(PutObjectInput { + bucket: "bkt".to_owned(), + key: "k".to_owned(), + ..Default::default() + })) + .await + .expect_err("a stalled fence must not be acknowledged"); + assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!( + dialed.load(Ordering::SeqCst), + 0, + "nothing reached the origin" + ); + assert_eq!(counter(&proxy, "ack_timeouts"), 1); + assert_eq!( + counter(&proxy, "feed_published"), + 2, + "the announcement and its abandonment" + ); +} + #[tokio::test] async fn an_unlicensed_read_records_its_bypass_reason() { let (_node, sync) = solo("unlicensed-read").await; diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 94954d8..03d4dd9 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -11,7 +11,10 @@ use groupnet::runtime::{NamedSeeds, ResolveFuture, SeedResolver, SystemResolver} use groupnet::transport::mem::Network; use s3s::dto::GetObjectOutput; -use crate::index::{KeyIndex, ObjEntry, standard_class}; +use crate::index::{ + AuthoritativeKeyState, KeyIndex, ObjEntry, resolve_uncertain_key, standard_class, + uncertain_key_is_current, +}; use crate::metrics::Metrics; use crate::sync::coherence::{ CAP_BOUNDED, Consistency, DEFAULT_LEASE_MS, KeyReconcile, WriteReceipt, WriteSync, WriteWait, @@ -644,8 +647,7 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { let ckey = ("bkt".to_owned(), "ptr".to_owned()); cache.insert(ckey.clone(), cached(b"old")).await; - let (_, publishing) = - sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", None, &metrics); + let (_, publishing) = sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", None, &metrics); let unknown = publishing.await; assert!( sync_b @@ -677,6 +679,194 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { assert_eq!(indexed_size(&state, "bkt", "ptr"), Some(5)); } +fn fenced(state: &Index, key: &str) -> bool { + state.read().unwrap().key_fenced("bkt", key) +} + +/// Until `writer` waits on `reader`'s serve-lease and not on its acks alone, a fence +/// could resolve without the reader in it. +async fn adopted(writer: &WriteSync, reader: &NodeId) { + eventually( + || { + writer.lease_holders() == std::slice::from_ref(reader) + && !waits_on_unleased(&writer.group, reader) + }, + "the writer to adopt the reader's serve-lease and its lease advertisement", + ) + .await; +} + +/// The pre-write fence: once it returns, the peer answers the key from the origin, and +/// the event reporting that write's outcome is what lifts it, for that write alone. +#[tokio::test] +async fn a_fenced_write_holds_the_peer_off_the_key_until_its_outcome_arrives() { + let net = Network::new(); + let (sync_a, sync_b, state, _cache) = + wired_pair_named(&net, ("fence-a", "fence-b"), Consistency::Strong).await; + adopted(&sync_a, &NodeId::new("fence-b")).await; + let metrics = Metrics::default(); + let own = KeyIndex::default(); + let first = own_put(&sync_a, "k", written(4), &metrics).await; + assert!( + sync_b + .reached_token(&first.header, Duration::from_secs(5)) + .await + ); + + let keys = ["k".to_owned(), "refused".to_owned()]; + let mut pending = sync_a + .fence(&own, "bkt", &keys, Duration::from_secs(5), &metrics) + .await + .expect("the peer acknowledges the fence"); + assert!( + fenced(&state, "k") && fenced(&state, "refused"), + "the peer holds the fence before the origin is called" + ); + let refused = pending.pop().flatten().expect("strong announces each key"); + let k = pending.pop().flatten().expect("strong announces each key"); + + let (_, publishing) = sync_a.index_put(&own, "bkt", "k", written(5), Some(&k), &metrics); + let outcome = publishing.await; + assert!( + sync_b + .reached_token(&outcome.header, Duration::from_secs(5)) + .await + ); + assert!(!fenced(&state, "k")); + assert_eq!(indexed_size(&state, "bkt", "k"), Some(5)); + assert!( + fenced(&state, "refused"), + "an outcome lifts only the write it settles" + ); + + sync_a + .index_abandoned(&own, "bkt", "refused", refused, &metrics) + .await; + eventually( + || !fenced(&state, "refused"), + "the abandonment lifts the fence", + ) + .await; + assert_eq!(indexed_size(&state, "bkt", "refused"), None); +} + +/// A writer that dies after its fence, before its origin call or after it, cannot say +/// what the origin holds. Its peer keeps the keys origin-only, and the writer's next +/// life hands them to the origin reconciliation. A write it never fenced never reached +/// the origin, so it leaves nothing to reconcile. +#[tokio::test] +async fn a_writer_that_dies_after_its_fence_leaves_the_keys_to_the_origin() { + let net = Network::new(); + let (a_id, a_node, a_group) = spawn_node(&net, "died-a", "died-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "died-b", "died-a").await; + let metrics = Arc::new(Metrics::default()); + let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); + let state = Arc::new(KeyIndex::default()); + let reconciling = Arc::new(std::sync::Mutex::new(Vec::new())); + let sync_b = Arc::new(attach(b_group, b_id.clone(), Consistency::Strong)); + let handed = Arc::clone(&reconciling); + sync_b.start_apply( + cache.local(), + state.clone(), + metrics.clone(), + Arc::new(move |bucket: &str, key: &str, token: u64| { + handed + .lock() + .unwrap() + .push((bucket.to_owned(), key.to_owned(), token)); + }), + ); + let sync_a = attach(a_group, a_id, Consistency::Strong); + adopted(&sync_a, &b_id).await; + + // `unsent` dies before its origin call; `applied` after the origin applied it. + let keys = ["unsent".to_owned(), "applied".to_owned()]; + let pending = sync_a + .fence( + &KeyIndex::default(), + "bkt", + &keys, + Duration::from_secs(5), + &metrics, + ) + .await + .expect("the peer acknowledges the fence"); + // The crash: no outcome and no seal. + drop(pending); + drop(sync_a); + a_node.close().await; + drop(a_node); + assert!(fenced(&state, "unsent") && fenced(&state, "applied")); + assert!(reconciling.lock().unwrap().is_empty()); + + // The pod returns under the same id as a new life, and writes another key. + let (a_id, _a_node, a_group) = spawn_node(&net, "died-a", "died-b").await; + let sync_a = attach(a_group, a_id, Consistency::Strong); + own_put(&sync_a, "other", written(1), &metrics).await; + eventually( + || reconciling.lock().unwrap().len() == 2, + "the new life hands both keys to the origin reconciliation", + ) + .await; + let handed: Vec<(String, String, u64)> = reconciling.lock().unwrap().clone(); + assert!( + handed.iter().all(|(_, key, _)| key != "other"), + "an unfenced write leaves nothing to reconcile" + ); + assert!(fenced(&state, "unsent") && fenced(&state, "applied")); + // What the origin answers resolves each: the old object, and the applied one. + for (bucket, key, token) in handed { + assert!(uncertain_key_is_current(&state, &bucket, &key, token)); + let size = if key == "applied" { 5 } else { 4 }; + assert!(resolve_uncertain_key( + &state, + &bucket, + &key, + token, + AuthoritativeKeyState::Present(written(size)), + )); + } + assert!(!fenced(&state, "unsent") && !fenced(&state, "applied")); + assert_eq!(indexed_size(&state, "bkt", "applied"), Some(5)); + assert_eq!(indexed_size(&state, "bkt", "unsent"), Some(4)); +} + +/// A peer cut off from the writer cannot hold the fence up: its serve-lease lapses on +/// the writer's clock, and the fence completes with the guarantee intact. +#[tokio::test] +async fn a_partitioned_peer_lapses_and_the_fence_completes() { + let net = Network::new(); + let (a_id, _a_node, a_group) = spawn_node(&net, "cut-a", "cut-b").await; + let (b_id, b_node, b_group) = spawn_node(&net, "cut-b", "cut-a").await; + let sync_a = attach(a_group, a_id, Consistency::Strong); + let _sync_b = attach(b_group, b_id.clone(), Consistency::Strong); + adopted(&sync_a, &b_id).await; + let metrics = Metrics::default(); + + // The cut: B's process runs on, but nothing reaches it or leaves it. + b_node.close().await; + let started = Instant::now(); + let pending = sync_a + .fence( + &KeyIndex::default(), + "bkt", + &["k".to_owned()], + Duration::from_millis(200), + &metrics, + ) + .await + .expect("a lapse carries the guarantee"); + assert!(pending[0].is_some()); + let waited = started.elapsed(); + assert!( + waited < TEST_LEASE * 3, + "the fence ends at the lapse, not at its own deadline ({waited:?})" + ); + let text = metrics.prometheus_text(); + assert!(text.contains("\ns3cache_write_lease_lapses 1\n")); + assert!(text.contains("\ns3cache_ack_timeouts 0\n")); +} + /// The strict-LIST barrier: after a publish, `await_fresh` on the peer /// returns true only once the event is actually applied. #[tokio::test] From 263ded0478b87b7f11aa7808029c4e26bb69f6db Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:08:26 -0600 Subject: [PATCH 16/19] test(strong_rw): report acknowledged write latency --- tests/it/strong_rw.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/it/strong_rw.rs b/tests/it/strong_rw.rs index 536182c..0f7beef 100644 --- a/tests/it/strong_rw.rs +++ b/tests/it/strong_rw.rs @@ -823,6 +823,22 @@ impl Cluster { } } let _ = writeln!(text, "write failures: {failures:?}"); + let mut latencies: Vec = history + .writes + .iter() + .filter(|write| write.status == WriteStatus::Acked) + .map(|write| write.done - write.invoked) + .collect(); + latencies.sort(); + if let (Some(median), Some(p99)) = ( + latencies.get(latencies.len() / 2), + latencies.get(latencies.len() * 99 / 100), + ) { + let _ = writeln!( + text, + "acknowledged write latency: median {median:?}, p99 {p99:?}" + ); + } for violation in violations.iter().take(40) { let _ = writeln!(text, " [{}] {}", violation.class, violation.text); } From c35e1b4e76ec6be58926db6c7e3a837b8ad6b55b Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:11:00 -0600 Subject: [PATCH 17/19] docs: describe the strong pre-write fence and its cost --- AGENTS.md | 12 +++++++++--- README.md | 26 +++++++++++++++++++------- src/sync/coherence.rs | 12 ++++++------ src/sync/coherence/apply.rs | 3 ++- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d593e0..eaeadb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,14 +46,16 @@ src/ mod.rs # cache link point: module docs and declarations only proxy.rs # cache types and core tier/index helpers service.rs # the S3 service implementation - mutation.rs # write-through paths: mutation tails, and unknown outcomes fenced and reconciled + mutation.rs # write-through paths: the strong pre-write fence, mutation tails, and unknown outcomes fenced and reconciled tests.rs # cache unit tests index.rs # in-memory LIST key index + index/pending.rs # peers' announced writes, fenced until settled tier.rs # hot (moka heap) / warm (mmap disk) tiered object-body cache sync/ mod.rs # sync link point: module docs and declarations only coherence.rs # gossip write-feed and coherence core coherence/apply.rs # the apply loop: peer writes into the index, acks, gaps and lapses + coherence/fence.rs # the writer's pre-write fence: announce, wait, settle or abandon config.rs # gossip env/config parsing and managed node construction volatile.rs # S3 origin facts/effects for Groupnet's bounded recovery driver wire.rs # write-event and session-token codec @@ -206,8 +208,12 @@ completion of the initial index and coherence warm-up. The default `strong` mode is **lease-backed**: a node may answer a LIST, a GET, or an authoritative 404 from its own state only while it holds an unexpired *serve-lease* its -peers granted (groupnet's `consistency::lease`, tier T3), and a write ends when every -lease-holder has either applied the invalidation (one ack round) or had its lease lapse +peers granted (groupnet's `consistency::lease`, tier T3). A write is **fenced** before +its origin call: it publishes `IndexOp::Pending` and waits until every lease-holder has +applied it or lapsed (`WriteSync::fence`), and each peer answers the key from the origin +until the outcome event settles it (`index::pending`); a writer's next life hands +anything it left open to the origin reconciliation. A write ends when every +lease-holder has either applied the outcome (one ack round) or had its lease lapse in the writer's engine (bounded by `D`, on the straggler's own clock). That replaces the old view-stability heuristic (`Stability`/`settled()`), which survives only for `strong-acks` — deprecated, removed next release. `strong-acks` pins the **mechanism** diff --git a/README.md b/README.md index 588bcde..7b6c539 100644 --- a/README.md +++ b/README.md @@ -213,13 +213,16 @@ heuristic earlier releases used: positive, not an absence oracle, and `false` covers everything: booting, warming up, lapsed, awaiting a resync, a peer gone silent, or a partition. Every one of them sends the read to the origin: slower, never stale (`unhealthy_bypasses`). -- **Writes end at an ack *or* at a lapse.** A write returns only after the origin acked - it AND every lease-holder either applied the invalidation (the fast path — one - in-cluster ack round, ~2 gossip hops, behind the origin round-trip already paid) or - had its serve-lease **expire in the writer's own engine** (the slow path — bounded by - `D`, counted as `write_lease_lapses`). So `PUT via A; GET via B` is deterministically - fresh, and a peer that stops answering costs one write up to `D` **once**, not an ack - timeout on every write forever. +- **Writes are fenced first, and end at an ack *or* at a lapse.** A write first + announces itself on the write feed and waits until every lease-holder holds that + announcement (or has lapsed); only then is it sent to the origin. It returns only + after the origin acked it AND every lease-holder either applied the outcome (the fast + path — one in-cluster ack round, ~2 gossip hops) or had its serve-lease **expire in + the writer's own engine** (the slow path — bounded by `D`, counted as + `write_lease_lapses`). A write pays that round twice, once on each side of the origin + round-trip. So `PUT via A; GET via B` is deterministically fresh, and a peer that + stops answering costs one write up to `D` **once**, not an ack timeout on every write + forever. - **That second path is the whole point.** The old mode ended an unacked write in a *degradation*: the writer proceeded and correctness then depended on the stale peer *learning* it should stand down — exactly what an asymmetrically-partitioned peer @@ -317,6 +320,15 @@ box is the long form): rule indexes that event skeletally, which also sends the key's reads to the origin. Every mutation runs on a task of its own, so a client hanging up mid-write cannot strand an applied DELETE, copy or completion outside the index. +- **A writer that dies mid-write leaves its keys on the origin.** From a write's fence + until its outcome arrives, every peer answers the key, and any LIST whose prefix + covers it, from the origin. If the writer crashes in between, before its origin call + or after the origin applied it, the peers keep doing so until the writer's next life + reaches them (a feed gap or a renewal); each such key is then unknown, and one origin + HEAD resolves it. A fence that does not reach every lease-holder by the deadline is + answered **503** without calling the origin (`ack_timeouts`). A write the origin + refused abandons its fence. A node holding a peer's open fence offers no index image + to a joining node. - **A feed gap discards the gapped node's index.** The node missed writes it can no longer order its rows against, so it keeps nothing for a peer-image install to carry forward; the install or the origin rebuild repopulates it. diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index 671ec82..f38a819 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -34,9 +34,9 @@ use crate::sync::wire::{ IndexEvent, IndexOp, encode_event, etag_to_wire, from_micros, parse_token, to_micros, }; -/// Ring capacity: a peer that falls further behind than this many writes -/// gets a gap (Groupnet closes serving, distrusts bodies, and rescans origin) instead -/// of per-event application. +/// Ring capacity: a peer that falls further behind than this many feed events (a +/// `strong` write publishes two) gets a gap (Groupnet closes serving, distrusts +/// bodies, and rescans origin) instead of per-event application. pub(super) const FEED_CAPACITY: usize = 4096; /// A peer feed position as the index's native writer cut. @@ -128,9 +128,9 @@ pub enum Consistency { /// whereas a lapse here is a guarantee, because a lapsed reader serves /// nothing cached until it re-synchronizes and affirms it. /// - /// Costs one ack round (~2 in-cluster hops, behind the origin round-trip - /// already paid) per write, plus one ledger-entry republish per applied - /// event and one small renewal entry per node per `D/3` — right for + /// Costs two ack rounds (~2 in-cluster hops each) per write, one to fence the + /// key before the origin call and one for its outcome, plus one ledger-entry + /// republish per applied event and one small renewal entry per node per `D/3` — right for /// fleet-sized clusters, wrong past the scaling envelope (run `bounded` /// and cells). The named failure modes are in the README's consistency /// section, and in groupnet's `consistency::lease` honesty box. diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs index 70639e6..ceedea5 100644 --- a/src/sync/coherence/apply.rs +++ b/src/sync/coherence/apply.rs @@ -15,7 +15,8 @@ use crate::tier::LocalCache; impl WriteSync { /// Spawn the apply loop: peer events update the LIST index and invalidate /// hot bodies. A peer write that describes nothing fences its key, and - /// `reconcile` resolves that fence from the origin. A gap synchronously + /// `reconcile` resolves that fence from the origin; a pending one fences it + /// until its writer settles it, or its next life hands it to `reconcile`. A gap synchronously /// closes Groupnet's local serving gate, discards the index, and schedules /// its rebuild. In `strong`, a small watcher reports the lease's monotone /// lapse count to that same recovery driver. From f2eb517f5ba7ceaeb35f8facab99986ea7f93d3e Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:31:52 -0600 Subject: [PATCH 18/19] test(coherence): count the fence and outcome events of each strong write --- tests/it/coherence.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/it/coherence.rs b/tests/it/coherence.rs index d5b7845..eb6cdb3 100644 --- a/tests/it/coherence.rs +++ b/tests/it/coherence.rs @@ -238,14 +238,18 @@ async fn sustained_writes_keep_the_coherence_feed_moving() { timeouts, "the coherence feed arrested at sustained write {seq}" ); + // A strong write publishes its fence and then its outcome. assert_eq!( counter(&metrics_b, "feed_applied"), - applied + seq + 1, + applied + 2 * (seq + 1), "strong mode returned write {seq} before B applied it" ); } - assert_eq!(counter(&metrics_a, "feed_published"), published + WRITES); + assert_eq!( + counter(&metrics_a, "feed_published"), + published + 2 * WRITES + ); assert_eq!( counter(&metrics_a, "write_lease_lapses"), lapses, From 56a1a7c0c485ff4a9f9c622a3ae338ace41cd32e Mon Sep 17 00:00:00 2001 From: Jon-Luke Biddle Date: Thu, 12 Nov 2026 04:31:53 -0600 Subject: [PATCH 19/19] docs: count feed events, and describe pending fences across fleet bootstrap --- docs/fleet-bootstrap.md | 18 ++++++++++-------- src/metrics.rs | 5 +++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/fleet-bootstrap.md b/docs/fleet-bootstrap.md index d34b8cc..69dd1a4 100644 --- a/docs/fleet-bootstrap.md +++ b/docs/fleet-bootstrap.md @@ -147,10 +147,11 @@ LIST/HEAD callbacks. A row carries exact key, size when known, `Last-Modified` as the donor holds it (see Follower handoff), `ETag`, and storage class. Peer rows deliberately become **skeletal**: `Content-Type` and user HEAD metadata are not fabricated; HEAD falls through to origin. The -donor cannot offer an image while any unresolved per-key uncertainty exists. -A follower carries its own unresolved uncertainty across the swap with its -reconciliation token (see Follower handoff); such reads keep going to the -origin until exact HEAD reconciliation or origin rebuild. +donor cannot offer an image while any unresolved per-key uncertainty, or any +peer's pending `strong` write, exists. A follower carries its own unresolved +uncertainty and the pending writes it holds across the swap (see Follower +handoff); such reads keep going to the origin until exact HEAD reconciliation, +the writer's outcome, or origin rebuild. Absence and LIST can be served locally only after a complete image, continuous native handoff, independent frontier check, and the existing lease/domain gate. Direct external S3 mutations still lack a fleet event and remain outside @@ -322,10 +323,11 @@ was written after that delete. Each live effect is then exactly one of: provably later, and timestamp ties never silently choose arrival order. Open reconciliations cross the swap with their exact tokens and the bucket's -token epoch. The fenced key stays origin-read, and its bucket's LIST stays -origin-served, until that key's own HEAD resolves it in the installed index or -a later definitive mutation supersedes it, as on a node that never -bootstrapped. No later fence can reuse a carried token. +token epoch. The fenced key stays origin-read, and a LIST whose prefix covers it +stays origin-served, until that key's own HEAD resolves it in the installed index +or a later definitive mutation supersedes it, as on a node that never +bootstrapped. No later fence can reuse a carried token. Pending peer writes cross +the swap unchanged and are settled by their writers' later events. **No live effect is lost.** A native effect the barrier does not cover misaligns the cuts and answers `NativePending`; the stage waits for a later diff --git a/src/metrics.rs b/src/metrics.rs index c4ff286..306ccaa 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -260,9 +260,10 @@ counters! { body_revalidation_etag_mismatch => body_revalidation_etag_mismatch, /// Record a suspect body whose indexed modification time is newer. body_revalidation_timestamp_mismatch => body_revalidation_timestamp_mismatch, - /// Record a write advertised to peers over the gossip write feed. + /// Record an event advertised to peers over the gossip write feed (a `strong` + /// write advertises its fence and its outcome). feed_published => feed_published, - /// Record a peer's write applied from the feed (index + invalidation). + /// Record a peer's feed event applied (index + invalidation). feed_applied => feed_applied, /// Record a feed gap (missed writes): every local body distrusted, index resynced. feed_gaps => feed_gap,