diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae34d32..5090a23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,18 @@ on: - ".dockerignore" - "deploy/**" - ".github/workflows/build.yml" + # Pull requests run the test gate only; the image push and the Fleet bump stay on main. + pull_request: + paths: + - "src/**" + - "tests/**" + - "scripts/**" + - "Cargo.toml" + - "Cargo.lock" + - "Dockerfile" + - ".dockerignore" + - "deploy/**" + - ".github/workflows/build.yml" workflow_dispatch: permissions: @@ -30,7 +42,8 @@ permissions: jobs: # Locked Rust lint/test/docs gate plus Helm lint/render. Run the real MinIO tests - # locally. CI runs unit tests and the integration modules that do not need an origin. + # locally. CI runs every unit test and the integration modules that do not need an + # origin. The unit tests run in their own step: a test-name filter would skip them. test: runs-on: ubuntu-latest permissions: @@ -44,7 +57,8 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all --check - run: cargo clippy --locked --workspace --all-targets --all-features - - run: "cargo test --locked --workspace --all-features --lib --bins --test it -- metrics_endpoint:: tier_cache:: startup_readiness::" + - run: cargo test --locked --workspace --all-features --lib --bins + - run: "cargo test --locked --workspace --all-features --test it -- metrics_endpoint:: tier_cache:: startup_readiness::" - run: cargo doc --locked --workspace --all-features --no-deps env: RUSTDOCFLAGS: -D warnings @@ -68,6 +82,7 @@ jobs: # reuse across builds. provenance/sbom off: the attestation stages turn the export # into a manifest list and break registry cache reuse (build-push-action#820). server: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read @@ -104,6 +119,7 @@ jobs: # The tag never moves backwards: an older run that finishes after a newer one keeps # the newer tag. bump: + if: github.event_name != 'pull_request' needs: [test, server] runs-on: ubuntu-latest permissions: diff --git a/AGENTS.md b/AGENTS.md index c2d7909..eaeadb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,17 +46,21 @@ src/ mod.rs # cache link point: module docs and declarations only proxy.rs # cache types and core tier/index helpers service.rs # the S3 service implementation - mutation.rs # write-through paths: mutation tails, and unknown outcomes fenced and reconciled + mutation.rs # write-through paths: the strong pre-write fence, mutation tails, and unknown outcomes fenced and reconciled tests.rs # cache unit tests index.rs # in-memory LIST key index + index/pending.rs # peers' announced writes, fenced until settled tier.rs # hot (moka heap) / warm (mmap disk) tiered object-body cache sync/ mod.rs # sync link point: module docs and declarations only coherence.rs # gossip write-feed and coherence core - config.rs # gossip env/config parsing and node construction + coherence/apply.rs # the apply loop: peer writes into the index, acks, gaps and lapses + coherence/fence.rs # the writer's pre-write fence: announce, wait, settle or abandon + config.rs # gossip env/config parsing and managed node construction volatile.rs # S3 origin facts/effects for Groupnet's bounded recovery driver wire.rs # write-event and session-token codec tests.rs # coherence unit tests + tests/fixtures.rs # managed in-memory nodes and wired pairs for those tests metrics.rs # counters, warm-residency gauges, and periodic stats logging tests/it/ # one integration-test binary; each module is a suite (links the library crate) deploy/helm/s3cache/ # Helm chart @@ -163,6 +167,11 @@ documentation-only edit does not require an image build. | `S3CACHE_STATS_SECS` | `60` | Stats log interval | | `S3CACHE_METRICS_LISTEN` | empty (disabled) | Bind address for the Prometheus text endpoint (`GET /metrics`) | +Gossip uses one managed network: `S3CACHE_GOSSIP_SEEDS` explicitly admits UDP routing +neighbors on both endpoints; DNS and advertisements refresh addresses only. The chart +lists every ordinal. Startup awaits link binding, and planned shutdown closes the +links and resolver after mutation drain and feed seal. + `GET /ready` on the metrics listener is the pod readiness probe (`CachingProxy::probe_ready`): ready once this node's index is complete, or while no live (`Alive`/`Suspect`) peer declares the `s3cache:indexed` gossip capability or @@ -199,8 +208,12 @@ completion of the initial index and coherence warm-up. The default `strong` mode is **lease-backed**: a node may answer a LIST, a GET, or an authoritative 404 from its own state only while it holds an unexpired *serve-lease* its -peers granted (groupnet's `consistency::lease`, tier T3), and a write ends when every -lease-holder has either applied the invalidation (one ack round) or had its lease lapse +peers granted (groupnet's `consistency::lease`, tier T3). A write is **fenced** before +its origin call: it publishes `IndexOp::Pending` and waits until every lease-holder has +applied it or lapsed (`WriteSync::fence`), and each peer answers the key from the origin +until the outcome event settles it (`index::pending`); a writer's next life hands +anything it left open to the origin reconciliation. A write ends when every +lease-holder has either applied the outcome (one ack round) or had its lease lapse in the writer's engine (bounded by `D`, on the straggler's own clock). That replaces the old view-stability heuristic (`Stability`/`settled()`), which survives only for `strong-acks` — deprecated, removed next release. `strong-acks` pins the **mechanism** diff --git a/Cargo.lock b/Cargo.lock index 5c19f7d..aee551e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1308,7 +1308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1541,11 +1541,14 @@ dependencies = [ [[package]] name = "groupnet" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "groupnet-consistency", "groupnet-core", + "groupnet-messaging", + "groupnet-network", "groupnet-runtime", + "groupnet-streams", "groupnet-transport", "groupnet-transport-mem", "groupnet-transport-tcp", @@ -1555,7 +1558,7 @@ dependencies = [ [[package]] name = "groupnet-consistency" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "bytes", "futures-util", @@ -1568,34 +1571,90 @@ dependencies = [ [[package]] name = "groupnet-core" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" + +[[package]] +name = "groupnet-messaging" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "bytes", + "groupnet-core", + "groupnet-network", + "groupnet-transport", + "ring", + "tokio", + "tokio-util", + "zerocopy", +] + +[[package]] +name = "groupnet-network" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "bytes", + "futures-util", + "groupnet-core", + "groupnet-transport", + "ring", + "rustls 0.23.45", + "tokio", + "tokio-rustls 0.26.6", + "tokio-util", + "zerocopy", +] [[package]] name = "groupnet-runtime" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" +dependencies = [ + "groupnet-core", + "groupnet-messaging", + "groupnet-network", + "groupnet-streams", + "groupnet-transport", + "tokio", + "tokio-util", +] + +[[package]] +name = "groupnet-streams" +version = "0.1.0" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", + "futures-util", "groupnet-core", + "groupnet-network", "groupnet-transport", + "ring", "tokio", + "tokio-util", + "zerocopy", ] [[package]] name = "groupnet-transport" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ "bytes", "futures-util", "groupnet-core", + "ring", + "tokio", + "tokio-util", "zerocopy", ] [[package]] name = "groupnet-transport-mem" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", @@ -1605,22 +1664,26 @@ dependencies = [ [[package]] name = "groupnet-transport-tcp" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", "tokio-util", + "zerocopy", ] [[package]] name = "groupnet-transport-udp" version = "0.1.0" -source = "git+https://github.com/napbat/groupnet?rev=fb97452387e396c9b77e91f15c654c69489e2378#fb97452387e396c9b77e91f15c654c69489e2378" +source = "git+https://github.com/napbat/groupnet?rev=30407c0bb057a7f12bf1520397f7398b38f97103#30407c0bb057a7f12bf1520397f7398b38f97103" dependencies = [ + "bytes", "groupnet-core", "groupnet-transport", "tokio", + "zerocopy", ] [[package]] @@ -2822,7 +2885,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3052,7 +3115,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3122,7 +3185,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4004,6 +4067,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "libc", "pin-project-lite", "tokio", @@ -4388,7 +4452,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 339935d..6b269c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ blake3 = "1.8.5" bytes = { version = "1", features = ["serde"] } futures = "0.3" getrandom = "0.4" -groupnet = { git = "https://github.com/napbat/groupnet", rev = "fb97452387e396c9b77e91f15c654c69489e2378", features = ["consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } +groupnet = { git = "https://github.com/napbat/groupnet", rev = "30407c0bb057a7f12bf1520397f7398b38f97103", default-features = false, features = ["runtime", "consistency-leases", "consistency-volatile-recovery", "consistency-volatile-bootstrap-bulk", "dns", "tcp", "udp"] } http = "1" http-body-util = "0.1" # Already under hyper-util/s3s; direct so the metrics endpoint can build its own responses. @@ -106,6 +106,8 @@ debug = "line-tables-only" opt-level = 2 [dev-dependencies] +# Managed in-process control links are test-only; production binds UDP. +groupnet = { workspace = true, features = ["mem"] } # Real bincode 1 bytes: the fixtures that prove `codec::legacy` still reads what # earlier releases stored. Never a runtime dependency. bincode.workspace = true diff --git a/README.md b/README.md index fba4cd9..7b6c539 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,13 @@ The fast path stays in local memory (LIST is served from RAM; nothing remote is read path). Loss is **detected, never silent**: a peer that falls behind the feed's ring, or a peer restart, surfaces as a *gap* — the node distrusts every local body and resyncs its index from the origin, which is the authority the index caches, so a copy is served -again only once that index has proved it current. Only seeds need static addressing; -every other peer resolves through gossiped advertisements. The ring is bounded by both +again only once that index has proved it current. `S3CACHE_GOSSIP_SEEDS` explicitly +admits routing neighbors: configure both endpoints of every intended link. DNS and +gossiped advertisements refresh admitted neighbors' addresses; discovering another +member does not grant admission. The Helm chart configures every ordinal, including +later-starting peers. Groupnet starts one managed network asynchronously and closes +its links and resolver after the planned mutation drain and feed seal. +The ring is bounded by both event count and encoded bytes: sustained long-key writes shorten its replay window and surface the same explicit gap instead of growing one state entry past the transport envelope and silently arresting peer application. @@ -208,13 +213,16 @@ heuristic earlier releases used: positive, not an absence oracle, and `false` covers everything: booting, warming up, lapsed, awaiting a resync, a peer gone silent, or a partition. Every one of them sends the read to the origin: slower, never stale (`unhealthy_bypasses`). -- **Writes end at an ack *or* at a lapse.** A write returns only after the origin acked - it AND every lease-holder either applied the invalidation (the fast path — one - in-cluster ack round, ~2 gossip hops, behind the origin round-trip already paid) or - had its serve-lease **expire in the writer's own engine** (the slow path — bounded by - `D`, counted as `write_lease_lapses`). So `PUT via A; GET via B` is deterministically - fresh, and a peer that stops answering costs one write up to `D` **once**, not an ack - timeout on every write forever. +- **Writes are fenced first, and end at an ack *or* at a lapse.** A write first + announces itself on the write feed and waits until every lease-holder holds that + announcement (or has lapsed); only then is it sent to the origin. It returns only + after the origin acked it AND every lease-holder either applied the outcome (the fast + path — one in-cluster ack round, ~2 gossip hops) or had its serve-lease **expire in + the writer's own engine** (the slow path — bounded by `D`, counted as + `write_lease_lapses`). A write pays that round twice, once on each side of the origin + round-trip. So `PUT via A; GET via B` is deterministically fresh, and a peer that + stops answering costs one write up to `D` **once**, not an ack timeout on every write + forever. - **That second path is the whole point.** The old mode ended an unacked write in a *degradation*: the writer proceeded and correctness then depended on the stale peer *learning* it should stand down — exactly what an asymmetrically-partitioned peer @@ -241,7 +249,7 @@ heuristic earlier releases used: after the lapse (so every wait held against the old one is resolved — read per granter rather than off the confirmed watermark, which is a *min* over the roster and advances when the member pinning it merely departs), settles for the fabric's - entry-propagation bound (`2 × anti_entropy_interval`), checks that no peer which was + entry-propagation bound (`2 × gossip_interval`), checks that no peer which was live at the lapse has vanished from membership since, and then barriers on every peer's advertised feed head — re-running the vanished check once more afterwards, because the barrier waits and a reap can land inside that wait. Past the barrier the apply loop has @@ -312,6 +320,15 @@ box is the long form): rule indexes that event skeletally, which also sends the key's reads to the origin. Every mutation runs on a task of its own, so a client hanging up mid-write cannot strand an applied DELETE, copy or completion outside the index. +- **A writer that dies mid-write leaves its keys on the origin.** From a write's fence + until its outcome arrives, every peer answers the key, and any LIST whose prefix + covers it, from the origin. If the writer crashes in between, before its origin call + or after the origin applied it, the peers keep doing so until the writer's next life + reaches them (a feed gap or a renewal); each such key is then unknown, and one origin + HEAD resolves it. A fence that does not reach every lease-holder by the deadline is + answered **503** without calling the origin (`ack_timeouts`). A write the origin + refused abandons its fence. A node holding a peer's open fence offers no index image + to a joining node. - **A feed gap discards the gapped node's index.** The node missed writes it can no longer order its rows against, so it keeps nothing for a peer-image install to carry forward; the install or the origin rebuild repopulates it. @@ -507,7 +524,7 @@ Unrelated documentation-only edits do not require a container build. | `S3CACHE_DISK_CACHE` | (empty) | Directory for the warm disk tier; unset = no disk tier | | `S3CACHE_DISK_CACHE_BYTES` | `10737418240` (10 GB) | Warm disk tier capacity (bytes) | | `S3CACHE_GOSSIP_BIND` | (empty) | UDP bind for the gossip write feed (e.g. `0.0.0.0:7946`); unset = single-node, no coherence layer | -| `S3CACHE_GOSSIP_SEEDS` | (empty) | Comma-separated seed peers as `id=host:port`; only seeds need static addressing | +| `S3CACHE_GOSSIP_SEEDS` | (empty) | Explicit UDP routing neighbors as comma-separated `id=host:port` pairs; configure both endpoints (self entries are ignored) | | `S3CACHE_GOSSIP_ADVERTISE` | bind addr | Address peers should dial back (set under NAT/container networking) | | `S3CACHE_CONSISTENCY` | `strong` | `strong`: lease-backed — a node serves locally only while it holds a serve-lease, and a write ends at an ack or at a lapse. `strong-acks`: the pre-lease ack-only mechanism, **deprecated**, removed next release. `bounded`: ~one-hop freshness, no per-write ack round (large clusters; set uniformly) | | `S3CACHE_LEASE_MS` | `2000` | Coherence-lease duration `D` (`strong` only): the writer's worst-case stall on a silent peer, each reader's window between confirmations, and the fleet's membership `dead_timeout`. Renewal traffic is one small entry per node per `D/3` | diff --git a/deploy/helm/s3cache/templates/statefulset.yaml b/deploy/helm/s3cache/templates/statefulset.yaml index a783a24..c924cda 100644 --- a/deploy/helm/s3cache/templates/statefulset.yaml +++ b/deploy/helm/s3cache/templates/statefulset.yaml @@ -90,11 +90,10 @@ spec: - name: S3CACHE_METRICS_LISTEN value: "0.0.0.0:{{ .Values.metrics.port | int64 }}" {{- end }} - # Pod name = gossip node id (HOSTNAME is set by the kubelet). Pod-0 is the - # only static seed; the rest resolves via gossiped advertisements. The - # advertise address must be the routable pod IP (peers parse it as an - # address), hence the downward API rather than a DNS name. Always on: a - # single replica is a cluster of one, and scaling is just replicaCount. + # Pod name = gossip node id (HOSTNAME is set by the kubelet). Every + # ordinal explicitly admits the other routing neighbors below. The + # advertise address is the routable pod IP, supplied by the downward API. + # Always on: a single replica is a cluster of one; scaling is replicaCount. - name: POD_IP valueFrom: fieldRef: @@ -103,12 +102,14 @@ spec: value: "0.0.0.0:{{ .Values.gossip.port }}" - name: S3CACHE_GOSSIP_ADVERTISE value: "$(POD_IP):{{ .Values.gossip.port }}" - # Two seed ordinals when there can be a peer: each pod skips itself, - # leaving pod-0 seeding pod-1 and everyone else seeding pod-0. A - # single-replica cluster names only pod-0, so its skipped self entry - # does not generate a permanent unresolved-peer warning. + # Explicitly admit every ordinal, including peers that have not started. + # Each pod skips itself; a singleton therefore has no unresolved seed. + {{- $gossipPeers := list }} + {{- range $ordinal := until (int .Values.replicaCount) }} + {{- $gossipPeers = append $gossipPeers (printf "s3cache-%d=s3cache-%d.s3cache-gossip:%d" $ordinal $ordinal (int $.Values.gossip.port)) }} + {{- end }} - name: S3CACHE_GOSSIP_SEEDS - value: {{ if gt (int .Values.replicaCount) 1 }}"s3cache-0=s3cache-0.s3cache-gossip:{{ .Values.gossip.port }},s3cache-1=s3cache-1.s3cache-gossip:{{ .Values.gossip.port }}"{{ else }}"s3cache-0=s3cache-0.s3cache-gossip:{{ .Values.gossip.port }}"{{ end }} + value: {{ join "," $gossipPeers | quote }} # Peer index bootstrap: a joining pod copies a ready peer's verified index # instead of listing the origin. Every ordinal shares this StatefulSet's # origin identity and bucket inventory; with no configured buckets there diff --git a/deploy/helm/s3cache/values.yaml b/deploy/helm/s3cache/values.yaml index d249115..e42aa0f 100644 --- a/deploy/helm/s3cache/values.yaml +++ b/deploy/helm/s3cache/values.yaml @@ -81,10 +81,10 @@ diskCache: volume: # any pod-volume spec backing diskCache.path emptyDir: {} -# Cross-node coherence via the gossip write feed (groupnet) — always on, no extra -# services, no toggle: pods gossip over a headless UDP service, pod-0 is the only -# static seed (everyone else resolves through gossiped advertisements), and pod names -# are the node identities. A single replica is a cluster of one. +# Cross-node coherence via managed gossip links (groupnet) — always on, no extra +# services, no toggle: each pod explicitly admits every ordinal over the headless +# UDP service. DNS and gossip refresh addresses, never neighbor admission. Pod names +# are node identities; a single replica is a cluster of one. gossip: port: 7946 # strong (default): coherence leases. A pod may answer from its own index or body cache diff --git a/docs/fleet-bootstrap.md b/docs/fleet-bootstrap.md index d34b8cc..69dd1a4 100644 --- a/docs/fleet-bootstrap.md +++ b/docs/fleet-bootstrap.md @@ -147,10 +147,11 @@ LIST/HEAD callbacks. A row carries exact key, size when known, `Last-Modified` as the donor holds it (see Follower handoff), `ETag`, and storage class. Peer rows deliberately become **skeletal**: `Content-Type` and user HEAD metadata are not fabricated; HEAD falls through to origin. The -donor cannot offer an image while any unresolved per-key uncertainty exists. -A follower carries its own unresolved uncertainty across the swap with its -reconciliation token (see Follower handoff); such reads keep going to the -origin until exact HEAD reconciliation or origin rebuild. +donor cannot offer an image while any unresolved per-key uncertainty, or any +peer's pending `strong` write, exists. A follower carries its own unresolved +uncertainty and the pending writes it holds across the swap (see Follower +handoff); such reads keep going to the origin until exact HEAD reconciliation, +the writer's outcome, or origin rebuild. Absence and LIST can be served locally only after a complete image, continuous native handoff, independent frontier check, and the existing lease/domain gate. Direct external S3 mutations still lack a fleet event and remain outside @@ -322,10 +323,11 @@ was written after that delete. Each live effect is then exactly one of: provably later, and timestamp ties never silently choose arrival order. Open reconciliations cross the swap with their exact tokens and the bucket's -token epoch. The fenced key stays origin-read, and its bucket's LIST stays -origin-served, until that key's own HEAD resolves it in the installed index or -a later definitive mutation supersedes it, as on a node that never -bootstrapped. No later fence can reuse a carried token. +token epoch. The fenced key stays origin-read, and a LIST whose prefix covers it +stays origin-served, until that key's own HEAD resolves it in the installed index +or a later definitive mutation supersedes it, as on a node that never +bootstrapped. No later fence can reuse a carried token. Pending peer writes cross +the swap unchanged and are settled by their writers' later events. **No live effect is lost.** A native effect the barrier does not cover misaligns the cuts and answers `NativePending`; the stage waits for a later diff --git a/scripts/resilience_e2e.py b/scripts/resilience_e2e.py index ec806a8..eb070f5 100755 --- a/scripts/resilience_e2e.py +++ b/scripts/resilience_e2e.py @@ -71,7 +71,7 @@ FAST_MS = 500 # How long the returning node gets to settle before anything is asserted about it: its -# lease warm-up (one detection window plus two anti-entropy rounds) and its bootstrap LIST. +# lease warm-up (one detection window plus two gossip rounds) and its bootstrap LIST. # Below that it is correct but origin-served, which is exactly what the polls tolerate. REJOIN_SETTLE_S = LEASE_MS / 1000.0 + 1.0 diff --git a/src/cache/copy.rs b/src/cache/copy.rs index b5ea6c8..dc75fb5 100644 --- a/src/cache/copy.rs +++ b/src/cache/copy.rs @@ -117,8 +117,11 @@ pub(super) fn indexed_source_size( return None; }; let state = proxy.state.read().unwrap(); + if state.key_fenced(bucket.as_ref(), key.as_ref()) { + return None; + } let bucket = state.get(bucket.as_ref())?; - if !bucket.synced || bucket.uncertain_keys.contains_key(key.as_ref()) { + if !bucket.synced { return None; } let entry = bucket.keys.get(key.as_ref())?; diff --git a/src/cache/mutation.rs b/src/cache/mutation.rs index bf60c7a..e4d9816 100644 --- a/src/cache/mutation.rs +++ b/src/cache/mutation.rs @@ -3,11 +3,14 @@ //! //! Each mutation's origin call and coherence tail run on a task of their own //! ([`CachingProxy::mutation_tail`]), so a client that hangs up cannot strand an applied -//! write outside the index. A success indexes and advertises what was written and holds -//! the response until the cluster has applied it. A failure that does not prove the -//! origin refused the write leaves the key's state unknown: it is fenced on every node -//! and reconciled from the origin ([`CachingProxy::publish_unknown`]) before the error -//! is answered, so no node answers that key from local state in the meantime. +//! write outside the index. In `strong`, the write is first announced to every peer +//! ([`WriteSync::fence`](crate::sync::coherence::WriteSync::fence)), so a writer that +//! dies after the origin applied it leaves no peer answering the key from local state. +//! A success indexes and advertises what was written and holds the response until the +//! cluster has applied it. A failure that does not prove the origin refused the write +//! leaves the key's state unknown: it is fenced on every node and reconciled from the +//! origin ([`CachingProxy::publish_unknown`]) before the error is answered. A refusal +//! abandons the announcement. use std::collections::BTreeSet; use std::sync::Arc; @@ -22,9 +25,11 @@ use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; use crate::cache::copy; use crate::cache::proxy::{ - CachingProxy, IndexedWrite, observed_entry, write_storage_class, written_object, + CachingProxy, IndexedWrite, WRITE_ACK_TIMEOUT, observed_entry, write_storage_class, + written_object, }; use crate::index::{ObjEntry, ObjMeta}; +use crate::sync::coherence::PendingWrite; use crate::tier::CachedObject; /// Why a failed mutation cannot be assumed not to have reached durable origin state. A @@ -59,6 +64,58 @@ fn copy_conflict_needs_reconcile(error: &S3Error, create_only: bool) -> bool { } impl CachingProxy { + /// Announce a write of `keys` to every peer before it reaches the origin. A stalled + /// announcement is a 503, and nothing reached the origin. + async fn fence_writes( + &self, + bucket: &str, + keys: &[String], + ) -> S3Result>> { + let Some(sync) = self.sync.as_ref() else { + return Ok(keys.iter().map(|_| None).collect()); + }; + sync.fence(&self.state, bucket, keys, WRITE_ACK_TIMEOUT, &self.metrics) + .await + .map_err(|_| { + s3s::s3_error!( + ServiceUnavailable, + "a cache peer could not be told about the write in time; it was not \ + sent to the origin, retry it" + ) + }) + } + + async fn fence_write(&self, bucket: &str, key: &str) -> S3Result> { + let mut pending = self.fence_writes(bucket, &[key.to_owned()]).await?; + Ok(pending.pop().flatten()) + } + + /// Settle a write the origin answered with an error: fenced and reconciled when the + /// error does not prove the origin refused it (`uncertain`), abandoned when it does. + async fn settle_failed( + &self, + bucket: &str, + key: &str, + uncertain: Option<&str>, + pending: Option, + ) { + match uncertain { + Some(reason) => { + self.publish_unknown(bucket, key, reason, pending.as_ref()) + .await; + } + None => self.abandon(bucket, key, pending).await, + } + } + + /// Settle a pending write that left the key as it was. + async fn abandon(&self, bucket: &str, key: &str, pending: Option) { + if let (Some(sync), Some(pending)) = (self.sync.as_ref(), pending) { + sync.index_abandoned(&self.state, bucket, key, pending, &self.metrics) + .await; + } + } + /// Forward a `PutObject`, then update the index from the result — and, when the write /// knows exactly what a read of it will report, keep the body it just wrote rather /// than dropping it (see [`buffered_put_body`](CachingProxy::buffered_put_body)). @@ -122,12 +179,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.put_object(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_mutation_error(&error, locally_vouched) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_mutation_error(&error, locally_vouched); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -149,7 +208,7 @@ impl CachingProxy { worker.metrics.write_fill(); } let token = worker - .record_put(IndexedWrite::Put, &bucket, &key, entry) + .record_put(IndexedWrite::Put, &bucket, &key, entry, pending.as_ref()) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) @@ -168,12 +227,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.delete_object(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_mutation_error(&error, false) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_mutation_error(&error, false); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -187,10 +248,10 @@ impl CachingProxy { // from the origin until an origin HEAD says. let receipt = if versioned { worker - .announce_unknown(&bucket, &key, "a version-scoped delete") + .announce_unknown(&bucket, &key, "a version-scoped delete", pending.as_ref()) .await } else { - worker.record_del(&bucket, &key).await + worker.record_del(&bucket, &key, pending.as_ref()).await }; let token = worker.await_cluster(receipt, &bucket, &key).await?; Self::attach_token(&mut resp.headers, token); @@ -216,21 +277,27 @@ impl CachingProxy { let keys: Vec = requested.iter().map(|(key, _)| key.clone()).collect(); let worker = self.clone(); let tail_bucket = bucket.clone(); + let fenced = keys.clone(); self.mutation_tail(&tail_bucket, &keys, async move { + let pending = worker.fence_writes(&bucket, &fenced).await?; let mut resp = match worker.inner.delete_objects(req).await { Ok(resp) => resp, Err(error) => { if let Some(reason) = uncertain_mutation_error(&error, false) { let mut receipt = None; - for (key, _) in &requested { + for ((key, _), pending) in requested.iter().zip(pending) { receipt = worker - .announce_unknown(&bucket, key, reason) + .announce_unknown(&bucket, key, reason, pending.as_ref()) .await .or(receipt); } worker .settle_cluster(receipt, &bucket, "") .await; + } else { + for ((key, _), pending) in requested.iter().zip(pending) { + worker.abandon(&bucket, key, pending).await; + } } return Err(error); } @@ -256,13 +323,14 @@ impl CachingProxy { .filter_map(|d| d.key.as_deref()) .collect(); let mut receipt = None; - for (key, versioned) in &requested { + for ((key, versioned), pending) in requested.iter().zip(pending) { let applied = if quiet { !refused.contains(key.as_str()) } else { deleted.contains(key.as_str()) }; if !applied { + worker.abandon(&bucket, key, pending).await; continue; } worker @@ -275,10 +343,10 @@ impl CachingProxy { // version is not the key (see `delete`): its key's state is unknown. let published = if *versioned { worker - .announce_unknown(&bucket, key, "a version-scoped delete") + .announce_unknown(&bucket, key, "a version-scoped delete", pending.as_ref()) .await } else { - worker.record_del(&bucket, key).await + worker.record_del(&bucket, key, pending.as_ref()).await }; receipt = published.or(receipt); } @@ -301,12 +369,14 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match worker.inner.complete_multipart_upload(req).await { Ok(resp) => resp, Err(error) => { - if let Some(reason) = uncertain_completion_error(&error) { - worker.publish_unknown(&bucket, &key, reason).await; - } + let uncertain = uncertain_completion_error(&error); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; return Err(error); } }; @@ -327,7 +397,13 @@ impl CachingProxy { let mut entry = observed_entry(observed.as_ref()); entry.etag = completed.or(entry.etag); let token = worker - .record_put(IndexedWrite::MultipartComplete, &bucket, &key, entry) + .record_put( + IndexedWrite::MultipartComplete, + &bucket, + &key, + entry, + pending.as_ref(), + ) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) @@ -353,10 +429,12 @@ impl CachingProxy { let worker = self.clone(); let (tail_bucket, tail_key) = (bucket.clone(), key.clone()); self.mutation_tail(&tail_bucket, &[tail_key], async move { + let pending = worker.fence_write(&bucket, &key).await?; let mut resp = match copy::forward(&worker.copy_inner, req).await { Ok(resp) => resp, Err(error) => { if copy_conflict_needs_reconcile(&error, create_only) { + worker.abandon(&bucket, &key, pending).await; // The origin's 412 proves an immutable create-only // destination already exists. That can race ahead of this // proxy's LIST index (for example after an interrupted @@ -375,8 +453,11 @@ impl CachingProxy { } else { worker.metrics.copy_conflict_reconcile_miss(); } - } else if let Some(reason) = uncertain_mutation_error(&error, false) { - worker.publish_unknown(&bucket, &key, reason).await; + } else { + let uncertain = uncertain_mutation_error(&error, false); + worker + .settle_failed(&bucket, &key, uncertain, pending) + .await; } return Err(error); } @@ -419,7 +500,7 @@ impl CachingProxy { }; entry.etag = copied_etag.or(entry.etag); let token = worker - .record_put(IndexedWrite::Copy, &bucket, &key, entry) + .record_put(IndexedWrite::Copy, &bucket, &key, entry, pending.as_ref()) .await?; Self::attach_token(&mut resp.headers, token); Ok(resp) diff --git a/src/cache/proxy.rs b/src/cache/proxy.rs index dcce9d3..3c14200 100644 --- a/src/cache/proxy.rs +++ b/src/cache/proxy.rs @@ -19,7 +19,10 @@ use crate::index::{ uncertain_key_is_current, }; use crate::metrics::Metrics; -use crate::sync::coherence::{READ_TOKEN_HEADER, WRITE_TOKEN_HEADER, WriteReceipt, WriteSync}; +use crate::sync::coherence::{ + PendingWrite, READ_TOKEN_HEADER, WRITE_TOKEN_HEADER, WriteReceipt, WriteSync, +}; +use crate::sync::config::RecoveryConfigRejected; use crate::sync::wire::wire_stamp; use crate::tier::{self, CachedObject, TieredCache, WarmPair}; use http::{HeaderMap, HeaderName, HeaderValue}; @@ -85,7 +88,7 @@ impl CachedRange { /// uniform leased fleet): the lease wait runs to its own `D + 1s` deadline, and the two /// run concurrently, so the response is held for the longer of the two — see /// [`WriteSync::wait_cluster_applied`](crate::sync::coherence::WriteSync). -const WRITE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +pub(super) const WRITE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); /// How long a strict read waits for the freshness barrier before failing closed to the /// origin. A healthy cluster resolves this immediately. @@ -505,22 +508,26 @@ impl CachingProxy { } /// Set finite recovery budgets and roster caps for gossip mode. - /// The settle window must still cover two local anti-entropy periods. + /// The settle window must still cover two local gossip rounds. /// /// # Errors - /// Returns an invalid-configuration error for zero/overflowing bounds or - /// an undersized gossip settle window. + /// Returns the rule a zero or overflowing bound breaks, or an undersized gossip + /// settle window. pub fn with_recovery_config( mut self, config: groupnet::consistency::volatile_recovery::RecoveryConfig, - ) -> Result { - config.validate()?; - if self + ) -> Result { + config.validate().map_err(RecoveryConfigRejected::Invalid)?; + if let Some(floor_ms) = self .sync .as_ref() - .is_some_and(|sync| config.settle_ms < sync.recovery_settle_floor_ms()) + .map(|sync| sync.recovery_settle_floor_ms()) + .filter(|floor_ms| config.settle_ms < *floor_ms) { - return Err(groupnet::consistency::volatile_recovery::RecoveryError::InvalidConfig); + return Err(RecoveryConfigRejected::SettleBelowGossip { + settle_ms: config.settle_ms, + floor_ms, + }); } self.recovery_config = Some(config); Ok(self) @@ -534,8 +541,9 @@ impl CachingProxy { pub fn with_recovery_rearm( mut self, policy: groupnet::consistency::volatile_recovery::RecoveryRearm, - ) -> Result { - self.recovery_rearm = Some(policy.validate()?); + ) -> Result { + policy.validate()?; + self.recovery_rearm = Some(policy); Ok(self) } @@ -736,7 +744,7 @@ impl CachingProxy { let Some(bucket) = g.get(input.bucket.as_str()).filter(|bucket| bucket.synced) else { return false; }; - if bucket.uncertain_keys.contains_key(input.key.as_str()) { + if g.key_fenced(&input.bucket, &input.key) { return false; } let entry = bucket.keys.get(input.key.as_str()); @@ -780,9 +788,15 @@ impl CachingProxy { /// until the cluster has fenced it too or lost the right to serve, exactly as a /// describable write is held ([`record_put`](Self::record_put)). A stalled wait /// changes nothing here: the caller's error already tells the client the outcome is - /// unknown. - pub(super) async fn publish_unknown(&self, bucket: &str, key: &str, reason: &str) { - let receipt = self.announce_unknown(bucket, key, reason).await; + /// unknown. `pending` is the write's own announcement, which this settles. + pub(super) async fn publish_unknown( + &self, + bucket: &str, + key: &str, + reason: &str, + pending: Option<&PendingWrite>, + ) { + let receipt = self.announce_unknown(bucket, key, reason, pending).await; self.settle_cluster(receipt, bucket, key).await; } @@ -794,6 +808,7 @@ impl CachingProxy { bucket: &str, key: &str, reason: &str, + pending: Option<&PendingWrite>, ) -> Option { let ckey = (bucket.to_owned(), key.to_owned()); let Some(sync) = self.sync.as_ref() else { @@ -801,7 +816,8 @@ impl CachingProxy { self.obj_cache.invalidate(&ckey).await; return None; }; - let (token, publishing) = sync.index_unknown(&self.state, bucket, key, &self.metrics); + let (token, publishing) = + sync.index_unknown(&self.state, bucket, key, pending, &self.metrics); spawn_key_reconcile(&self.client, &self.state, bucket, key, token, reason); // Fenced first, so no fill can commit the old body behind this: a copy left in // place would be served again, trusted, once the reconciliation resolves. @@ -809,14 +825,10 @@ impl CachingProxy { Some(publishing.await) } - /// A key fenced by an ambiguous mutation may only be read from the origin until its - /// exact reconciliation token is resolved or superseded. - fn key_uncertain(&self, bucket: &str, key: &str) -> bool { - self.state - .read() - .unwrap() - .get(bucket) - .is_some_and(|bucket| bucket.uncertain_keys.contains_key(key)) + /// A key fenced by an ambiguous mutation, or by a peer's pending write, may only be + /// read from the origin until that fence is resolved or settled. + fn key_fenced(&self, bucket: &str, key: &str) -> bool { + self.state.read().unwrap().key_fenced(bucket, key) } /// Attach a write's session token to a response, when coherence is on. @@ -842,20 +854,26 @@ impl CachingProxy { /// coherence is on, or a 503 when the wait stalled (see /// [`await_cluster`](Self::await_cluster)). The caller has already dropped this /// node's own body copy — the writer must not serve its own stale bytes while the - /// cluster round runs. + /// cluster round runs. `pending` is the write's own announcement, which this settles. pub(super) async fn record_put( &self, op: IndexedWrite, bucket: &str, key: &str, mut entry: ObjEntry, + pending: Option<&PendingWrite>, ) -> S3Result> { // A write whose response carried neither an `ETag` nor a size describes nothing // a reader could check a copy against: it is reconciled as an uncertain one. if entry.etag.is_none() && entry.size.is_none() { op.record(&self.metrics); let receipt = self - .announce_unknown(bucket, key, "the write's response described no object") + .announce_unknown( + bucket, + key, + "the write's response described no object", + pending, + ) .await; return self.await_cluster(receipt, bucket, key).await; } @@ -869,7 +887,8 @@ impl CachingProxy { } return Ok(None); }; - let (changed, publishing) = sync.index_put(&self.state, bucket, key, entry, &self.metrics); + let (changed, publishing) = + sync.index_put(&self.state, bucket, key, entry, pending, &self.metrics); if changed { op.record(&self.metrics); } @@ -880,15 +899,21 @@ impl CachingProxy { /// Record a durable delete: tombstone + remove locally and advertise it to peers, /// returning the receipt rather than waiting. The caller chooses when to wait — one /// key waits immediately, a batch publishes every key first and then waits once (see - /// [`await_cluster`](Self::await_cluster)). - pub(super) async fn record_del(&self, bucket: &str, key: &str) -> Option { + /// [`await_cluster`](Self::await_cluster)). `pending` is the write's own + /// announcement, which this settles. + pub(super) async fn record_del( + &self, + bucket: &str, + key: &str, + pending: Option<&PendingWrite>, + ) -> Option { let ts = wire_stamp(SystemTime::now()); let Some(sync) = self.sync.as_ref() else { apply_del(&self.state, bucket, key, ts); return None; }; Some( - sync.index_del(&self.state, bucket, key, ts, &self.metrics) + sync.index_del(&self.state, bucket, key, ts, pending, &self.metrics) .await, ) } @@ -1213,7 +1238,7 @@ impl CachingProxy { || { self.obj_cache.suspect_gen() == generation && self.sync.as_ref().is_none_or(|sync| sync.may_serve_local()) - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) }, |obj, observed| { self.observe(&ckey.0, &ckey.1, observed); @@ -1269,9 +1294,7 @@ impl CachingProxy { resume_after: Option<&str>, ) -> Option { let g = self.state.read().unwrap(); - if g.get(inp.bucket.as_str()) - .is_some_and(|bucket| !bucket.uncertain_keys.is_empty()) - { + if g.fenced_under(&inp.bucket, inp.prefix.as_deref().unwrap_or_default()) { return None; } list_objects_v2_from_index( @@ -1406,7 +1429,7 @@ impl CachingProxy { // overwrite is never read stale; a token-carrying read that cannot be // verified in time skips every local copy and streams from the origin. let local_ok = if cacheable || cached_range.is_some() { - !self.key_uncertain(&ckey.0, &ckey.1) + !self.key_fenced(&ckey.0, &ckey.1) && self.read_barrier(&req.headers).await == ReadRoute::Local } else { true @@ -1512,7 +1535,7 @@ impl CachingProxy { let can_commit = || { self.obj_cache.suspect_gen() == generation && self.sync.as_ref().is_none_or(|sync| sync.may_serve_local()) - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) }; let on_commit = |filled: &Arc, observed: &ObservedObject| { // The GET supplies both body and HEAD metadata. Fold the observation @@ -1565,7 +1588,7 @@ impl CachingProxy { && req.input.if_unmodified_since.is_none(); let ckey = (req.input.bucket.clone(), req.input.key.clone()); if cache_eligible - && !self.key_uncertain(&ckey.0, &ckey.1) + && !self.key_fenced(&ckey.0, &ckey.1) && self.read_barrier(&req.headers).await == ReadRoute::Local { if let Some(obj) = self.validated_get(&ckey).await diff --git a/src/cache/stop.rs b/src/cache/stop.rs index 88d03f5..bb71372 100644 --- a/src/cache/stop.rs +++ b/src/cache/stop.rs @@ -140,7 +140,7 @@ impl CachingProxy { let mut receipt = None; for key in keys { receipt = self - .announce_unknown(bucket, key, "the mutation task terminated") + .announce_unknown(bucket, key, "the mutation task terminated", None) .await .or(receipt); } @@ -196,6 +196,9 @@ impl CachingProxy { async { tokio::time::timeout(DRAIN_WAIT, drain).await.is_ok() }, self.seal_writes(SEAL_WAIT), ); + if let Some(sync) = &self.sync { + sync.close().await; + } if drained { info!("graceful shutdown complete"); } else { diff --git a/src/cache/tests.rs b/src/cache/tests.rs index 877ec37..406fabb 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -1,14 +1,20 @@ use std::num::NonZeroUsize; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, SystemTime}; use bytes::Bytes; -use groupnet::consistency::{LeaseConfig, WriteFeed, advertised_head}; +use groupnet::consistency::lease::LeaseConfig; +use groupnet::consistency::{WriteFeed, advertised_head}; use groupnet::core::{Config, NodeId}; +use groupnet::network::RouterConfig; use groupnet::runtime::Node; -use groupnet::transport::mem::{MemTransport, Network}; +use groupnet::transport::link::{BoundLink, LinkConfig}; +use groupnet::transport::mem::Network; use http::HeaderMap; -use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, ListObjectsV2Input, Timestamp}; +use s3s::dto::{ + ETag, GetObjectInput, GetObjectOutput, ListObjectsV2Input, PutObjectInput, Timestamp, +}; use s3s::{S3, S3ErrorCode, S3Request}; use crate::cache::proxy::{CacheConfig, CachingProxy, FullSyncOwner, ObservedObject, ReadRoute}; @@ -25,12 +31,11 @@ use crate::tier::CachedObject; /// warm-up window is milliseconds rather than a second. fn brisk() -> Config { Config { - gossip_interval_ms: 10, + gossip_interval_ms: 25, probe_interval_ms: 20, probe_timeout_ms: 10, suspect_timeout_ms: 50, dead_timeout_ms: 300, - anti_entropy_interval_ms: 25, ..Config::default() } } @@ -38,12 +43,18 @@ fn brisk() -> Config { /// One leased node, alone in its group. A solo reader's granter roster is empty, so /// its lease confirms vacuously — which is exactly why the shell's warm-up guard, /// not the confirmation, is what a booting node has to get past. -fn solo(id: &str) -> (Node, Arc) { +async fn solo(id: &str) -> (Node, Arc) { let net = Network::new(); let me = NodeId::new(id); - let node = Node::builder(me.clone(), net.endpoint(me.clone())) + let node = Node::builder(me.clone()) + .link(BoundLink::new( + net.endpoint(me.clone()), + LinkConfig::new(Vec::new()), + )) .config(brisk()) - .spawn(); + .start() + .await + .expect("solo managed node starts"); let group = node.join_group("s3cache"); let sync = WriteSync::attach( group, @@ -68,6 +79,11 @@ fn only_the_newest_single_node_warmup_may_retry() { /// A proxy over an origin it never dials. Every case below is decided from local /// state; a row that reached the endpoint would fail on the connection, not pass. fn proxy(sync: Option>) -> CachingProxy { + proxy_at("http://127.0.0.1:1", sync) +} + +/// A proxy whose origin is `endpoint`. +fn proxy_at(endpoint: &str, sync: Option>) -> CachingProxy { let conf = aws_sdk_s3::config::Builder::new() .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) .region(aws_sdk_s3::config::Region::new("us-east-1")) @@ -78,7 +94,7 @@ fn proxy(sync: Option>) -> CachingProxy { None, "s3cache-unit-tests", )) - .endpoint_url("http://127.0.0.1:1") + .endpoint_url(endpoint) .force_path_style(true) .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) .build(); @@ -123,7 +139,7 @@ async fn revoke_before_local_return( #[tokio::test] async fn a_lease_revoked_after_local_body_lookup_forces_origin_get() { - let (_node, sync) = solo("return-fence-get"); + let (_node, sync) = solo("return-fence-get").await; let proxy = proxy(Some(sync.clone())); let body = cached("v1", at(1_700_000_000)); body.mark_trusted(proxy.obj_cache.suspect_gen()); @@ -158,7 +174,7 @@ async fn a_lease_revoked_after_local_body_lookup_forces_origin_get() { #[tokio::test] async fn a_lease_revoked_after_index_answer_forces_origin_list() { - let (_node, sync) = solo("return-fence-list"); + let (_node, sync) = solo("return-fence-list").await; let proxy = proxy(Some(sync.clone())); index(&proxy, "k", Some("v1"), at(1_700_000_000)); synced(&proxy); @@ -294,7 +310,7 @@ fn counter(proxy: &CachingProxy, name: &str) -> u64 { /// body of that same version can pass revalidation after warm-up completes. #[tokio::test] async fn an_unknown_read_observation_keeps_the_origin_mtime() { - let (_node, sync) = solo("observed-origin-mtime"); + let (_node, sync) = solo("observed-origin-mtime").await; let proxy = proxy(Some(sync)); let modified = at(1_700_000_000); let observed = ObservedObject { @@ -336,14 +352,32 @@ async fn a_freshness_timeout_routes_to_origin_and_is_counted() { let net = Network::new(); let a_id = NodeId::new("barrier-a"); let b_id = NodeId::new("barrier-b"); - let a_node = Node::builder(a_id.clone(), net.endpoint(a_id.clone())) - .seed(b_id.clone()) + let a_node = Node::builder(a_id.clone()) + .link(BoundLink::new( + net.endpoint(a_id.clone()), + LinkConfig::new(vec![b_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); - let b_node = Node::builder(b_id.clone(), net.endpoint(b_id.clone())) - .seed(a_id.clone()) + .start() + .await + .expect("A managed node starts"); + let b_node = Node::builder(b_id.clone()) + .link(BoundLink::new( + net.endpoint(b_id.clone()), + LinkConfig::new(vec![a_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); + .start() + .await + .expect("B managed node starts"); let a_group = a_node.join_group("s3cache"); let b_group = b_node.join_group("s3cache"); let sync = Arc::new(WriteSync::attach( @@ -375,7 +409,7 @@ async fn a_freshness_timeout_routes_to_origin_and_is_counted() { NonZeroUsize::new(4).unwrap_or(NonZeroUsize::MIN), |_value: &u8| vec![0], ); - let token = feed.publish(&1).await; + let token = feed.publish(&1).await.expect("the feed is open"); tokio::time::timeout(Duration::from_secs(2), async { loop { if advertised_head(&b_group, &a_id) == Some(token) { @@ -404,14 +438,32 @@ async fn a_stalled_write_is_a_503_not_a_success() { let net = Network::new(); let a_id = NodeId::new("stall-a"); let b_id = NodeId::new("stall-b"); - let a_node = Node::builder(a_id.clone(), net.endpoint(a_id.clone())) - .seed(b_id.clone()) + let a_node = Node::builder(a_id.clone()) + .link(BoundLink::new( + net.endpoint(a_id.clone()), + LinkConfig::new(vec![b_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); - let b_node = Node::builder(b_id.clone(), net.endpoint(b_id.clone())) - .seed(a_id.clone()) + .start() + .await + .expect("A managed node starts"); + let b_node = Node::builder(b_id.clone()) + .link(BoundLink::new( + net.endpoint(b_id.clone()), + LinkConfig::new(vec![a_id.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) .config(brisk()) - .spawn(); + .start() + .await + .expect("B managed node starts"); let lease = LeaseConfig::for_duration(Duration::from_millis(300)); // A advertises the ack tier but runs no apply loop, so it never acknowledges. let _peer = WriteSync::attach( @@ -447,16 +499,107 @@ async fn a_stalled_write_is_a_503_not_a_success() { meta: None, }; let error = proxy - .record_put(crate::cache::proxy::IndexedWrite::Put, "bkt", "k", entry) + .record_put( + crate::cache::proxy::IndexedWrite::Put, + "bkt", + "k", + entry, + None, + ) .await .expect_err("a stalled write must not be acknowledged"); assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); assert_eq!(counter(&proxy, "ack_timeouts"), 1); } +async fn linked(net: &Network, id: &NodeId, peer: &NodeId) -> Node { + Node::builder(id.clone()) + .link(BoundLink::new( + net.endpoint(id.clone()), + LinkConfig::new(vec![peer.clone()]), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) + .config(brisk()) + .start() + .await + .expect("managed node starts") +} + +/// The fence comes before the origin call. A write whose fence cannot complete is a 503 +/// that never reached the origin, which is also all a writer that dies before its fence +/// completes can have done. The announcement is abandoned, not left open. +#[tokio::test] +async fn a_stalled_fence_is_a_503_that_never_reaches_the_origin() { + let origin = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("the origin binds"); + let endpoint = format!("http://{}", origin.local_addr().expect("origin address")); + let dialed = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&dialed); + tokio::spawn(async move { + while origin.accept().await.is_ok() { + counted.fetch_add(1, Ordering::SeqCst); + } + }); + let net = Network::new(); + let (a_id, b_id) = (NodeId::new("fence-stall-a"), NodeId::new("fence-stall-b")); + let a_node = linked(&net, &a_id, &b_id).await; + let b_node = linked(&net, &b_id, &a_id).await; + let lease = LeaseConfig::for_duration(Duration::from_millis(300)); + // A is in the ack tier without a lease and runs no apply loop: B's fence waits on + // it and never hears back. + let _peer = WriteSync::attach( + a_node.join_group("s3cache"), + a_id.clone(), + Consistency::StrongAcks, + lease, + None, + ); + let b_group = b_node.join_group("s3cache"); + let sync = Arc::new(WriteSync::attach( + b_group.clone(), + b_id, + Consistency::Strong, + lease, + None, + )); + let proxy = proxy_at(&endpoint, Some(sync)); + tokio::time::timeout(Duration::from_secs(2), async { + while !b_group.members().contains(&a_id) { + tokio::task::yield_now().await; + } + }) + .await + .expect("the peers converge"); + + let error = proxy + .put_object(request(PutObjectInput { + bucket: "bkt".to_owned(), + key: "k".to_owned(), + ..Default::default() + })) + .await + .expect_err("a stalled fence must not be acknowledged"); + assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!( + dialed.load(Ordering::SeqCst), + 0, + "nothing reached the origin" + ); + assert_eq!(counter(&proxy, "ack_timeouts"), 1); + assert_eq!( + counter(&proxy, "feed_published"), + 2, + "the announcement and its abandonment" + ); +} + #[tokio::test] async fn an_unlicensed_read_records_its_bypass_reason() { - let (_node, sync) = solo("unlicensed-read"); + let (_node, sync) = solo("unlicensed-read").await; let proxy = proxy(Some(sync)); assert!(matches!( proxy.read_barrier(&HeaderMap::new()).await, @@ -492,7 +635,7 @@ async fn a_single_node_serves_a_suspect_copy_unproved() { /// atomic load, without the index being asked anything. #[tokio::test] async fn a_proved_copy_is_served_without_consulting_the_index() { - let (_node, sync) = solo("proved"); + let (_node, sync) = solo("proved").await; let proxy = proxy(Some(sync)); let obj = cached("v1", at(1_700_000_000)); obj.mark_trusted(proxy.obj_cache.suspect_gen()); @@ -579,7 +722,7 @@ async fn peer_put_cannot_complete_new_index_metadata_from_an_older_get() { /// and not once per read. The second call must not touch the index again. #[tokio::test] async fn a_suspect_copy_the_index_confirms_is_proved_exactly_once() { - let (_node, sync) = solo("confirm"); + let (_node, sync) = solo("confirm").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); proxy.obj_cache.insert(ck("k"), cached("v1", filled)).await; @@ -602,7 +745,7 @@ async fn a_suspect_copy_the_index_confirms_is_proved_exactly_once() { /// drop it and let the origin serve the read. #[tokio::test] async fn a_suspect_copy_the_index_contradicts_is_dropped() { - let (_node, sync) = solo("contradict"); + let (_node, sync) = solo("contradict").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); synced(&proxy); @@ -649,7 +792,7 @@ async fn a_suspect_copy_the_index_contradicts_is_dropped() { /// entry is stamped microseconds after its body. #[tokio::test] async fn a_byte_identical_rewrite_is_caught_by_the_mtime_and_a_fresh_fill_is_not() { - let (_node, sync) = solo("rewrite"); + let (_node, sync) = solo("rewrite").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); synced(&proxy); @@ -694,7 +837,7 @@ async fn a_byte_identical_rewrite_is_caught_by_the_mtime_and_a_fresh_fill_is_not /// are actually read. #[tokio::test] async fn an_unsynced_bucket_arbitrates_nothing_and_drops_the_copy() { - let (_node, sync) = solo("unsynced"); + let (_node, sync) = solo("unsynced").await; let proxy = proxy(Some(sync)); let filled = at(1_700_000_000); proxy.obj_cache.insert(ck("k"), cached("v1", filled)).await; diff --git a/src/index.rs b/src/index.rs index ed20218..b69863e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -29,6 +29,7 @@ use s3s::dto::{ }; pub(crate) mod fleet; +pub(crate) mod pending; mod rows; mod scan; use rows::{GoneRows, KeyRows}; @@ -250,6 +251,7 @@ struct KeyIndexState { stats: IndexStats, capture: Option, native_cuts: BTreeMap, (u64, u64)>, + pending: pending::PendingWrites, } /// Read guard over the key map. The aggregate statistics share its lock but remain an diff --git a/src/index/fleet.rs b/src/index/fleet.rs index 56cbb14..3f90ca3 100644 --- a/src/index/fleet.rs +++ b/src/index/fleet.rs @@ -8,10 +8,13 @@ use std::collections::{BTreeMap, HashMap}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use groupnet::consistency::volatile_recovery::bootstrap::ports::{JournalIngress, LogicalClock}; -use groupnet::core::volatile_bootstrap::journal::{DeltaIdentity, Invalidation, NativeCut}; +use groupnet::core::volatile_bootstrap::journal::{ + DeltaIdentity, Invalidation, NativeCut, NativeRenewal, +}; use s3s::dto::{ETag, ObjectStorageClass}; +use super::pending::PendingWrites; use super::{BucketState, GoneRows, IndexStats, KeyIndexState, KeyRows, ObjEntry}; const VERSION: u8 = 1; @@ -165,9 +168,13 @@ impl IndexCapture { return; }; let now = self.clock.now(); + let renewal = NativeRenewal { + sealed: sealed.clone(), + epoch, + }; let _ = self .ingress - .with_journal(|journal| journal.renew(now, self.generation, sealed, epoch, effect)); + .with_journal(|journal| journal.renew(now, self.generation, renewal, effect)); } } @@ -949,6 +956,7 @@ pub(super) fn decode(bytes: &[u8], caps: ImageCaps) -> Result config.max_cuts { + if live.native_cuts.len() > config.max_cuts.get() { return Err(ImageError::Capacity); } cuts.try_reserve_exact(live.native_cuts.len()) .map_err(|_| ImageError::Capacity)?; - if cuts.capacity() > config.max_cuts.saturating_mul(2).max(4) { + if cuts.capacity() > config.max_cuts.get().saturating_mul(2).max(4) { return Err(ImageError::Capacity); } let mut cut_bytes = 0usize; @@ -214,14 +225,14 @@ impl KeyIndex { cut_bytes = cut_bytes .checked_add(writer.len()) .ok_or(ImageError::Capacity)?; - if writer.is_empty() || cut_bytes > config.max_cut_bytes { + if writer.is_empty() || cut_bytes > config.max_cut_bytes.get() { return Err(ImageError::Capacity); } let mut identity = Vec::new(); identity .try_reserve_exact(writer.len()) .map_err(|_| ImageError::Capacity)?; - if identity.capacity() > config.max_cut_bytes { + if identity.capacity() > config.max_cut_bytes.get() { return Err(ImageError::Capacity); } identity.extend_from_slice(writer); @@ -234,10 +245,11 @@ impl KeyIndex { journal .begin_capture( clock.now(), - size.encoded_bytes, - size.decoded_bytes, - members, - cuts, + CapturePlan { + image, + members, + cuts, + }, ) .map_err(|_| ImageError::Incomplete)?; let ingress = @@ -249,7 +261,7 @@ impl KeyIndex { ingress.clone(), generation, clock, - config.max_event_bytes, + config.max_event_bytes.get(), caps.name_bytes, )); let lock_held = locked.elapsed(); @@ -324,24 +336,28 @@ mod tests { use super::*; + fn n(value: usize) -> NonZeroUsize { + NonZeroUsize::new(value).unwrap() + } + fn config() -> JournalConfig { JournalConfig { - max_encoded_bytes: 2_048, - max_decoded_bytes: 8_192, - max_events: 4, - max_suffix_bytes: 1_024, - max_event_bytes: 128, - max_identity_bytes: 64, - max_followers: 1, - max_follower_id_bytes: 64, - max_cuts: 2, - max_cut_bytes: 128, - max_members: 2, - max_membership_bytes: 256, - max_scope_bytes: 128, - max_batch_events: 1, - max_batch_bytes: 128, - max_inflight_bytes: 256, + max_encoded_bytes: n(2_048), + max_decoded_bytes: n(8_192), + max_events: n(4), + max_suffix_bytes: n(1_024), + max_event_bytes: n(128), + max_identity_bytes: n(64), + max_followers: n(1), + max_follower_id_bytes: n(64), + max_cuts: n(2), + max_cut_bytes: n(128), + max_members: n(2), + max_membership_bytes: n(256), + max_scope_bytes: n(128), + max_batch_events: n(1), + max_batch_bytes: n(128), + max_inflight_bytes: n(256), max_total_ms: 100_000, max_follower_ms: 10_000, } @@ -379,13 +395,13 @@ mod tests { fn admission() -> ByteAdmission { ByteAdmission::new(AdmissionLimits { - max_total_bytes: 1_000_000, + max_total_bytes: n(1_000_000), max_encoded_bytes: 16_384, max_decoded_bytes: 65_536, max_suffix_bytes: 900_000, max_native_overlap_bytes: 1, max_inflight_bytes: 1, - max_reservations: 16, + max_reservations: n(16), }) .unwrap() } @@ -471,7 +487,10 @@ mod tests { donor .ingress() .with_journal(|journal| journal.image_charge()) - .map(|charge| (charge.encoded_bytes, charge.decoded_bytes)), + .map(|charge| ( + charge.image.encoded_bytes.get(), + charge.image.decoded_bytes.get() + )), Some((size.encoded_bytes, size.decoded_bytes)) ); } @@ -538,7 +557,7 @@ mod tests { DonorJournal::storage_bound(config).unwrap(), ) .unwrap(); - let held = budget.reserve(AdmissionClass::Decoded, 65_536).unwrap(); + let held = budget.reserve(AdmissionClass::Decoded, n(65_536)).unwrap(); let before = budget.usage().0; let refused = index.begin_fleet_capture( journal, @@ -874,18 +893,18 @@ mod tests { name_bytes: 64, }; let config = JournalConfig { - max_encoded_bytes: caps.bytes, - max_decoded_bytes: caps.decoded_bytes, + max_encoded_bytes: n(caps.bytes), + max_decoded_bytes: n(caps.decoded_bytes), ..config() }; let budget = ByteAdmission::new(AdmissionLimits { - max_total_bytes: 1 << 28, + max_total_bytes: n(1 << 28), max_encoded_bytes: caps.bytes, max_decoded_bytes: caps.decoded_bytes, max_suffix_bytes: 900_000, max_native_overlap_bytes: 1, max_inflight_bytes: 1, - max_reservations: 16, + max_reservations: n(16), }) .unwrap(); let suffix = budget diff --git a/src/index/fleet/install.rs b/src/index/fleet/install.rs index a9a75a8..ee82736 100644 --- a/src/index/fleet/install.rs +++ b/src/index/fleet/install.rs @@ -476,6 +476,7 @@ impl KeyIndex { capture.invalidate(Invalidation::Rebuild); } candidate.native_cuts = std::mem::take(&mut live.native_cuts); + candidate.pending = std::mem::take(&mut live.pending); *live = candidate; Ok(()) } diff --git a/src/index/fleet/snapshot.rs b/src/index/fleet/snapshot.rs index a540d68..e30fe76 100644 --- a/src/index/fleet/snapshot.rs +++ b/src/index/fleet/snapshot.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, HashMap}; +use crate::index::pending::PendingWrites; use crate::index::{BucketState, IndexStats, KeyIndexState}; use super::{ @@ -12,8 +13,8 @@ use super::{ }; /// One universe bucket, named in strictly ascending order, completely synced -/// and free of uncertainty: the bucket-level refusals shared by the size at C -/// and the measurement off-lock. +/// and free of uncertainty and pending peer writes: the bucket-level refusals +/// shared by the size at C and the measurement off-lock. fn image_bucket<'a>( index: &'a KeyIndexState, name: &'a str, @@ -26,7 +27,11 @@ fn image_bucket<'a>( } *previous = Some(name); let bucket = index.buckets.get(name).ok_or(ImageError::Incomplete)?; - if !bucket.synced || !bucket.uncertain_keys.is_empty() || bucket.rebuild_generation.is_some() { + if !bucket.synced + || !bucket.uncertain_keys.is_empty() + || index.pending.fenced_under(name, "") + || bucket.rebuild_generation.is_some() + { return Err(ImageError::Incomplete); } Ok(bucket) @@ -124,6 +129,7 @@ pub(crate) fn snapshot_state( stats: total, capture: None, native_cuts: BTreeMap::new(), + pending: PendingWrites::default(), }) } diff --git a/src/index/fleet/stage.rs b/src/index/fleet/stage.rs index 6273f54..37c5419 100644 --- a/src/index/fleet/stage.rs +++ b/src/index/fleet/stage.rs @@ -335,7 +335,7 @@ mod tests { identity: DeltaIdentity::Local(b"repair".to_vec()), effect, }], - bytes: 42, + bytes: std::num::NonZeroUsize::new(42).unwrap(), }; stage.stage_batch(&batch).unwrap(); assert_eq!(stage.through(), &through); diff --git a/src/index/pending.rs b/src/index/pending.rs new file mode 100644 index 0000000..0e626da --- /dev/null +++ b/src/index/pending.rs @@ -0,0 +1,349 @@ +//! Peer writes announced before their origin call and not settled yet. +//! +//! A `strong` writer publishes [`IndexOp::Pending`](crate::sync::wire::IndexOp) and +//! waits for its peers before it calls the origin. A reader holding one answers the +//! key from the origin until the writer settles it: a writer that crashes after the +//! origin applied its write never says what it did. A settling event names its pending +//! write, and every event carries the writer's oldest one still open, so a settle lost +//! to a gap is covered by the next event. What a writer's ended life left open is +//! fenced as uncertain instead (see [`fence_uncertain_key`](super::fence_uncertain_key)): +//! only the origin can say what it did. + +use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound; + +use groupnet::core::volatile_bootstrap::journal::{DeltaIdentity, NativeCut}; + +use super::{KeyIndex, KeyIndexReadGuard, KeyIndexState}; + +/// A key fenced as uncertain, with the token its origin reconciliation must present. +pub(crate) type UncertainKey = (String, String, u64); + +/// One writer life's open pending writes, by feed sequence. +struct Life { + epoch: u64, + open: BTreeMap, +} + +#[derive(Default)] +pub(super) struct PendingWrites { + writers: HashMap, Life>, + /// How many open pending writes fence each key, per bucket. + keys: HashMap>, +} + +impl PendingWrites { + #[cfg(test)] + fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + fn fenced(&self, bucket: &str, key: &str) -> bool { + self.keys + .get(bucket) + .is_some_and(|keys| keys.contains_key(key)) + } + + pub(super) fn fenced_under(&self, bucket: &str, prefix: &str) -> bool { + self.keys + .get(bucket) + .is_some_and(|keys| first_under(keys, prefix)) + } + + fn hold(&mut self, writer: &[u8], epoch: u64, seq: u64, bucket: &str, key: &str) { + let life = self.writers.entry(writer.to_vec()).or_insert_with(|| Life { + epoch, + open: BTreeMap::new(), + }); + if life.epoch != epoch { + return; + } + let entry = (bucket.to_owned(), key.to_owned()); + if life.open.insert(seq, entry).is_none() { + *self + .keys + .entry(bucket.to_owned()) + .or_default() + .entry(key.to_owned()) + .or_default() += 1; + } + } + + /// Settle `settles` and everything below `floor` (below `seq`, the settling + /// event's own position, when the writer had nothing open). + fn settle( + &mut self, + writer: &[u8], + epoch: u64, + seq: u64, + settles: Option, + floor: Option, + ) { + let Some(life) = self.writers.get_mut(writer) else { + return; + }; + if life.epoch != epoch { + return; + } + let mut done: Vec<(String, String)> = settles + .and_then(|settled| life.open.remove(&settled)) + .into_iter() + .collect(); + let still_open = life.open.split_off(&floor.unwrap_or(seq)); + done.extend(std::mem::replace(&mut life.open, still_open).into_values()); + if life.open.is_empty() { + self.writers.remove(writer); + } + for (bucket, key) in &done { + self.release(bucket, key); + } + } + + /// End `writer`'s life if it is older than `epoch`, returning what it left open. + fn end_life_before(&mut self, writer: &[u8], epoch: u64) -> Vec<(String, String)> { + if self + .writers + .get(writer) + .is_none_or(|life| life.epoch >= epoch) + { + return Vec::new(); + } + let Some(life) = self.writers.remove(writer) else { + return Vec::new(); + }; + let ended: Vec<(String, String)> = life.open.into_values().collect(); + for (bucket, key) in &ended { + self.release(bucket, key); + } + ended + } + + fn release(&mut self, bucket: &str, key: &str) { + let Some(keys) = self.keys.get_mut(bucket) else { + return; + }; + if let Some(count) = keys.get_mut(key) { + *count -= 1; + if *count == 0 { + keys.remove(key); + } + } + if keys.is_empty() { + self.keys.remove(bucket); + } + } +} + +/// Whether any key of a sorted set starts with `prefix`. +fn first_under(keys: &BTreeMap, prefix: &str) -> bool { + keys.range::((Bound::Included(prefix), Bound::Unbounded)) + .next() + .is_some_and(|(key, _)| key.starts_with(prefix)) +} + +impl KeyIndexState { + /// Fence what an ended writer life left pending as uncertain. + fn end_pending_life(&mut self, writer: &[u8], epoch: u64) -> Vec { + self.pending + .end_life_before(writer, epoch) + .into_iter() + .map(|(bucket, key)| { + let token = self.fence_key(&bucket, &key); + (bucket, key, token) + }) + .collect() + } +} + +impl KeyIndex { + /// Hold a peer's pending write under the lock that notes its feed position. The + /// event has no index effect, so the donor journal records a no-op for it. + /// Returns the keys an ended life of that writer left open, now uncertain. + pub(crate) fn hold_pending_native( + &self, + bucket: &str, + key: &str, + cut: &NativeCut, + floor: Option, + ) -> Vec { + let mut index = self.inner.write().unwrap(); + if !index.note_native_cut(cut) { + return Vec::new(); + } + if let Some(capture) = index.capture.as_mut() { + capture.record_native_noop(DeltaIdentity::Native(cut.clone())); + } + let ended = index.end_pending_life(&cut.writer, cut.epoch); + index + .pending + .settle(&cut.writer, cut.epoch, cut.sequence, None, floor); + index + .pending + .hold(&cut.writer, cut.epoch, cut.sequence, bucket, key); + ended + } + + /// Settle a peer's pending writes as a later event of its feed says, after that + /// event's own index effect. An [`IndexOp::Abandoned`](crate::sync::wire::IndexOp) + /// has none, so `noop` notes its position and journals it here. Returns the keys + /// an ended life of that writer left open, now uncertain. + pub(crate) fn settle_pending_native( + &self, + cut: &NativeCut, + settles: Option, + floor: Option, + noop: bool, + ) -> Vec { + let mut index = self.inner.write().unwrap(); + if noop { + if !index.note_native_cut(cut) { + return Vec::new(); + } + if let Some(capture) = index.capture.as_mut() { + capture.record_native_noop(DeltaIdentity::Native(cut.clone())); + } + } + let ended = index.end_pending_life(&cut.writer, cut.epoch); + index + .pending + .settle(&cut.writer, cut.epoch, cut.sequence, settles, floor); + ended + } + + /// End every pending write of `writer`'s lives before `epoch`: its feed moved + /// past them (a new life, or a seal). Returns those keys, now uncertain. + pub(crate) fn end_pending_life(&self, writer: &[u8], epoch: u64) -> Vec { + self.inner.write().unwrap().end_pending_life(writer, epoch) + } + + /// Note this node's own feed event that changes no index row — its pending and + /// abandoned writes — under the lock that assigns its position, journaled as a + /// no-op so the donor journal stays contiguous. + pub(crate) fn note_own_native(&self, assign: impl FnOnce() -> (R, NativeCut)) -> R { + let mut index = self.inner.write().unwrap(); + let (value, position) = assign(); + if index.note_native_cut(&position) + && let Some(capture) = index.capture.as_mut() + { + capture.record_native_noop(DeltaIdentity::Native(position)); + } + value + } +} + +impl KeyIndexReadGuard<'_> { + /// Whether `key` may only be answered from the origin: its last mutation is + /// unreconciled, or a peer's write of it is pending. + pub(crate) fn key_fenced(&self, bucket: &str, key: &str) -> bool { + self.0.pending.fenced(bucket, key) + || self + .0 + .buckets + .get(bucket) + .is_some_and(|b| b.uncertain_keys.contains_key(key)) + } + + /// Whether any key under `prefix` is fenced (see [`key_fenced`](Self::key_fenced)). + pub(crate) fn fenced_under(&self, bucket: &str, prefix: &str) -> bool { + self.0.pending.fenced_under(bucket, prefix) + || self + .0 + .buckets + .get(bucket) + .is_some_and(|b| first_under(&b.uncertain_keys, prefix)) + } +} + +#[cfg(test)] +mod tests { + use groupnet::core::volatile_bootstrap::journal::NativeCut; + + use crate::index::{KeyIndex, uncertain_key_is_current}; + + fn cut(epoch: u64, sequence: u64) -> NativeCut { + NativeCut { + writer: b"writer".to_vec(), + epoch, + sequence, + } + } + + fn fenced(state: &KeyIndex, key: &str) -> bool { + state.read().unwrap().key_fenced("bkt", key) + } + + #[test] + fn a_settle_releases_only_the_pending_write_it_names() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "a", &cut(1, 1), None); + state.hold_pending_native("bkt", "b", &cut(1, 2), Some(1)); + state.settle_pending_native(&cut(1, 3), Some(2), Some(1), false); + assert!(fenced(&state, "a"), "a's write is still open"); + assert!(!fenced(&state, "b")); + state.settle_pending_native(&cut(1, 4), Some(1), None, false); + assert!(!fenced(&state, "a")); + assert!(state.read().unwrap().0.pending.is_empty()); + } + + /// Two writes of one key by one writer fence it until both settle. + #[test] + fn a_key_stays_fenced_while_any_write_of_it_is_open() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.hold_pending_native("bkt", "k", &cut(1, 2), Some(1)); + state.settle_pending_native(&cut(1, 3), Some(1), Some(2), false); + assert!(fenced(&state, "k")); + state.settle_pending_native(&cut(1, 4), Some(2), None, false); + assert!(!fenced(&state, "k")); + } + + /// The floor settles a write whose settling event a gap swallowed. + #[test] + fn the_floor_settles_a_write_whose_settle_was_lost() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "lost", &cut(1, 1), None); + state.hold_pending_native("bkt", "open", &cut(1, 2), Some(1)); + // Sequence 3 settled `lost` and never arrived; 4 says 2 is the oldest open. + state.settle_pending_native(&cut(1, 4), None, Some(2), true); + assert!(!fenced(&state, "lost")); + assert!(fenced(&state, "open")); + } + + /// A duplicate delivery changes nothing. + #[test] + fn a_duplicate_pending_event_is_held_once() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + state.settle_pending_native(&cut(1, 2), Some(1), None, true); + assert!(!fenced(&state, "k")); + } + + /// A writer's next life fences what its last one left open as uncertain: the + /// crashed life cannot say whether the origin applied it. + #[test] + fn a_new_life_turns_open_writes_into_uncertain_keys() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "k", &cut(1, 1), None); + assert!(state.end_pending_life(b"writer", 1).is_empty(), "same life"); + let ended = state.end_pending_life(b"writer", 2); + assert_eq!(ended.len(), 1); + let (bucket, key, token) = &ended[0]; + assert_eq!((bucket.as_str(), key.as_str()), ("bkt", "k")); + assert!(uncertain_key_is_current(&state, "bkt", "k", *token)); + assert!(state.read().unwrap().0.pending.is_empty()); + assert!(fenced(&state, "k"), "uncertain until the origin says"); + } + + #[test] + fn a_list_is_fenced_only_under_a_prefix_holding_a_fenced_key() { + let state = KeyIndex::default(); + state.hold_pending_native("bkt", "log/0002", &cut(1, 1), None); + let guard = state.read().unwrap(); + assert!(guard.fenced_under("bkt", "")); + assert!(guard.fenced_under("bkt", "log/")); + assert!(!guard.fenced_under("bkt", "ptr/")); + assert!(!guard.fenced_under("bkt", "log/0003")); + assert!(!guard.fenced_under("other", "")); + } +} diff --git a/src/metrics.rs b/src/metrics.rs index c4ff286..306ccaa 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -260,9 +260,10 @@ counters! { body_revalidation_etag_mismatch => body_revalidation_etag_mismatch, /// Record a suspect body whose indexed modification time is newer. body_revalidation_timestamp_mismatch => body_revalidation_timestamp_mismatch, - /// Record a write advertised to peers over the gossip write feed. + /// Record an event advertised to peers over the gossip write feed (a `strong` + /// write advertises its fence and its outcome). feed_published => feed_published, - /// Record a peer's write applied from the feed (index + invalidation). + /// Record a peer's feed event applied (index + invalidation). feed_applied => feed_applied, /// Record a feed gap (missed writes): every local body distrusted, index resynced. feed_gaps => feed_gap, diff --git a/src/sync/coherence.rs b/src/sync/coherence.rs index 4241c1b..f38a819 100644 --- a/src/sync/coherence.rs +++ b/src/sync/coherence.rs @@ -1,18 +1,26 @@ +mod apply; +mod fence; + +pub(crate) use fence::PendingWrite; + use std::num::NonZeroUsize; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime}; +use groupnet::consistency::lease::{ + CAP_LEASE, CoherenceOutcome, LeaseConfig, LeaseView, Leases, RenewalId, +}; use groupnet::consistency::volatile_recovery::{RecoveryError, RecoveryHandle, RecoveryStatus}; use groupnet::consistency::{ - AckLedger, CAP_ACKS, CAP_LEASE, CoherenceOutcome, Frontier, LeaseConfig, LeaseView, Leases, - PeerWrite, PeerWrites, RenewalId, WriteFeed, WriteToken, advertised_head, applied_by_selected, + AckWait, CAP_ACKS, PublishError, WriteFeed, WriteToken, advertised_head, applied_by_selected, }; use groupnet::core::volatile_bootstrap::journal::NativeCut; use groupnet::core::{NodeId, Status}; -use groupnet::runtime::Group; +use groupnet::runtime::{Group, Node}; use s3s::dto::ObjectStorageClass; use tracing::{info, warn}; +use crate::index::pending::UncertainKey; use crate::index::{ KeyIndex, ObjEntry, apply_del_native, apply_own_del, apply_own_put, apply_put_native, fence_own_uncertain_key, fence_uncertain_key_native, standard_class, @@ -23,14 +31,12 @@ use crate::sync::readiness::UnmetSeeds; use crate::sync::stop::Crossing; use crate::sync::volatile::CacheRecoveryAdapter; use crate::sync::wire::{ - IndexEvent, IndexOp, decode_event, encode_event, etag_to_wire, from_micros, parse_token, - to_micros, + IndexEvent, IndexOp, encode_event, etag_to_wire, from_micros, parse_token, to_micros, }; -use crate::tier::LocalCache; -/// Ring capacity: a peer that falls further behind than this many writes -/// gets a gap (Groupnet closes serving, distrusts bodies, and rescans origin) instead -/// of per-event application. +/// Ring capacity: a peer that falls further behind than this many feed events (a +/// `strong` write publishes two) gets a gap (Groupnet closes serving, distrusts +/// bodies, and rescans origin) instead of per-event application. pub(super) const FEED_CAPACITY: usize = 4096; /// A peer feed position as the index's native writer cut. @@ -43,20 +49,31 @@ pub(super) fn native_cut(peer: &NodeId, token: WriteToken) -> NativeCut { } /// Fold one peer event into the index. A put that describes nothing (see -/// [`IndexOp::unknown`]) fences its key instead, and the fence's token is returned for -/// the caller to reconcile with the origin. +/// [`IndexOp::unknown`]) fences its key instead; a pending write fences it until the +/// writer settles it. Returns every key left uncertain, for the caller to reconcile +/// with the origin. fn apply_feed_index( state: &KeyIndex, peer: &NodeId, token: WriteToken, event: &IndexEvent, -) -> Option { +) -> Vec { let ts = from_micros(event.ts_us); let cut = native_cut(peer, token); - if event.op.describes_nothing() { - return fence_uncertain_key_native(state, &event.bucket, &event.key, &cut); - } + let mut uncertain = Vec::new(); match &event.op { + IndexOp::Pending => { + return state.hold_pending_native(&event.bucket, &event.key, &cut, event.floor); + } + IndexOp::Abandoned => { + return state.settle_pending_native(&cut, event.settles, event.floor, true); + } + op if op.describes_nothing() => { + if let Some(fence) = fence_uncertain_key_native(state, &event.bucket, &event.key, &cut) + { + uncertain.push((event.bucket.clone(), event.key.clone(), fence)); + } + } IndexOp::Put { size, etag, @@ -75,13 +92,16 @@ fn apply_feed_index( content_type: content_type.clone(), meta: None, }; - apply_put_native(state, &event.bucket, &event.key, entry, cut); + apply_put_native(state, &event.bucket, &event.key, entry, cut.clone()); } IndexOp::Del => { - apply_del_native(state, &event.bucket, &event.key, ts, cut); + apply_del_native(state, &event.bucket, &event.key, ts, cut.clone()); } } - None + // The outcome is indexed first, so a read that no longer sees the pending write + // sees what replaced it. + uncertain.extend(state.settle_pending_native(&cut, event.settles, event.floor, false)); + uncertain } /// Starts an origin reconciliation of a key this node fenced under `token` (see @@ -108,9 +128,9 @@ pub enum Consistency { /// whereas a lapse here is a guarantee, because a lapsed reader serves /// nothing cached until it re-synchronizes and affirms it. /// - /// Costs one ack round (~2 in-cluster hops, behind the origin round-trip - /// already paid) per write, plus one ledger-entry republish per applied - /// event and one small renewal entry per node per `D/3` — right for + /// Costs two ack rounds (~2 in-cluster hops each) per write, one to fence the + /// key before the origin call and one for its outcome, plus one ledger-entry + /// republish per applied event and one small renewal entry per node per `D/3` — right for /// fleet-sized clusters, wrong past the scaling envelope (run `bounded` /// and cells). The named failure modes are in the README's consistency /// section, and in groupnet's `consistency::lease` honesty box. @@ -163,7 +183,8 @@ impl Consistency { } /// Whether this mode participates in the ack tier — publishes an - /// [`AckLedger`] and waits on other participants' watermarks. Exhaustive + /// [`AckLedger`](groupnet::consistency::AckLedger) and waits on participants' + /// watermarks. Exhaustive /// on purpose: a mode added later has to answer this question rather than /// inherit an answer from a `!= Bounded`. pub(super) fn acks(self) -> bool { @@ -354,10 +375,9 @@ pub struct WriteSync { /// The deadline the coherence wait gets: one lease duration (the longest a /// silent holder's lapse can take) plus [`WRITE_WAIT_SLACK`]. write_wait: Duration, - /// Keeps the gossip node (receive loop, group actors) alive for the - /// process lifetime, whatever its transport. `None` when a test drives a raw - /// group directly. - _node: Option>, + /// Owns the managed gossip network for this feed. `None` when a test holds + /// the node separately while driving its group. + pub(super) node: Option, /// This node's `~caps` declaration; [`WriteSync::advertise_indexed`] extends it. pub(super) advertisement: Arc, /// Seeds not in the roster yet (see [`WriteSync::expect_seeds`]). @@ -365,6 +385,9 @@ pub struct WriteSync { /// The counted granters membership last did not report alive, as last logged /// (see [`absent_granters`](Self::absent_granters)). absent_granters: std::sync::Mutex>, + /// The feed sequences of this node's pending writes not settled yet (see + /// [`fence`](Self::fence)). + open_writes: fence::OpenWrites, } impl WriteSync { @@ -389,7 +412,7 @@ impl WriteSync { me: NodeId, consistency: Consistency, lease: LeaseConfig, - node: Option>, + node: Option, ) -> Self { debug_assert!( lease.validate().is_ok(), @@ -398,9 +421,10 @@ impl WriteSync { ); let capacity = NonZeroUsize::new(FEED_CAPACITY).unwrap_or(NonZeroUsize::MIN); let feed = WriteFeed::new(group.clone(), capacity, encode_event); - let leases = consistency - .leases() - .then(|| Leases::new(group.clone(), me.clone(), lease)); + let leases = consistency.leases().then(|| { + Leases::new(group.clone(), me.clone(), lease) + .expect("a validated lease config inside a Tokio runtime") + }); let lease_view = leases.as_ref().map(Leases::view); let advertisement = Advertisement::declare(group.clone(), consistency); Self { @@ -418,7 +442,8 @@ impl WriteSync { advertisement, unmet_seeds: Arc::default(), absent_granters: std::sync::Mutex::default(), - _node: node, + open_writes: fence::OpenWrites::default(), + node, } } @@ -525,10 +550,7 @@ impl WriteSync { } pub(crate) fn recovery_settle_floor_ms(&self) -> u64 { - self.group - .config() - .anti_entropy_interval_ms - .saturating_mul(2) + self.group.config().gossip_interval_ms.saturating_mul(2) } pub(super) fn frontier_view(&self) -> Option { @@ -546,7 +568,7 @@ impl WriteSync { && self .lease_view .as_ref() - .is_none_or(groupnet::consistency::LeaseView::mark_caught_up) + .is_none_or(LeaseView::mark_caught_up) } /// The newest renewal of this node's serve-lease that **every** granter in its @@ -600,14 +622,16 @@ impl WriteSync { /// Index this node's own put and advertise it to peers: everything the entry /// carries that a peer can use, stamped with the local index's timestamp. The feed /// assigns the write's position under the index lock that applies it, so every own - /// write reaches the index, the donor journal and the feed in one order. Returns - /// whether the index changed and the advertisement to await. + /// write reaches the index, the donor journal and the feed in one order. `settles` + /// is the write's pending announcement, if it made one. Returns whether the index + /// changed and the advertisement to await. pub(crate) fn index_put<'a>( &'a self, state: &KeyIndex, bucket: &str, key: &str, entry: ObjEntry, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> (bool, impl Future + use<'a>) { let event = IndexEvent { @@ -620,9 +644,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(entry.last_modified), + settles: None, + floor: None, }; let (changed, publishing) = apply_own_put(state, bucket, key, entry, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); (changed, self.advertised(publishing, metrics)) } @@ -635,6 +661,7 @@ impl WriteSync { bucket: &str, key: &str, ts: SystemTime, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> impl Future + use<'a> { let event = IndexEvent { @@ -642,9 +669,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(ts), + settles: None, + floor: None, }; let (_, publishing) = apply_own_del(state, bucket, key, ts, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); self.advertised(publishing, metrics) } @@ -658,6 +687,7 @@ impl WriteSync { state: &KeyIndex, bucket: &str, key: &str, + settles: Option<&PendingWrite>, metrics: &'a Metrics, ) -> (u64, impl Future + use<'a>) { let event = IndexEvent { @@ -665,9 +695,11 @@ impl WriteSync { bucket: bucket.to_owned(), key: key.to_owned(), ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, }; let (token, publishing) = fence_own_uncertain_key(state, bucket, key, || { - (self.feed.publish(&event), self.own_position()) + self.publish_own(event, settles, false) }); (token, self.advertised(publishing, metrics)) } @@ -684,10 +716,21 @@ impl WriteSync { async fn advertised( &self, - publishing: impl Future, + publishing: impl Future>, metrics: &Metrics, ) -> WriteReceipt { - let token = publishing.await; + let token = match publishing.await { + Ok(token) => token, + // Recorded in the ring, which the next advertisement carries; until then no + // peer can apply it, so the write's ack wait decides whether it succeeded. + Err(PublishError::Unadvertised(unadvertised)) => { + warn!("{unadvertised}"); + unadvertised.token + } + Err(PublishError::Sealed) => { + panic!("published after the feed was sealed: the seal promised no more writes") + } + }; metrics.feed_published(); WriteReceipt { token, @@ -716,12 +759,15 @@ impl WriteSync { (Consistency::Strong | Consistency::StrongAcks, _) => { let acked = applied_by_selected( &self.group, - &self.me, - token, + AckWait { + writer: &self.me, + token, + timeout, + }, |node| waits_on(&self.group, node), - timeout, ) - .await; + .await + .is_complete(); if acked { WriteWait::Applied } else { @@ -757,13 +803,19 @@ impl WriteSync { ) -> WriteWait { let (coherent, insured) = tokio::join!( leases.invalidated_coherently(&self.me, token, self.write_wait), - applied_by_selected( - &self.group, - &self.me, - token, - |node| waits_on_unleased(&self.group, node), - timeout, - ), + async { + applied_by_selected( + &self.group, + AckWait { + writer: &self.me, + token, + timeout, + }, + |node| waits_on_unleased(&self.group, node), + ) + .await + .is_complete() + }, ); match (coherent, insured) { // A deadline the lease tier could not meet is the one outcome with no @@ -802,6 +854,27 @@ impl WriteSync { bucket: &str, key: &str, metrics: &Metrics, + ) -> bool { + self.acked( + token, + timeout, + bucket, + key, + metrics, + "answering 503, the write may be applied at the origin", + ) + .await + } + + /// [`ack_write`](Self::ack_write), logging `on_stall` when the wait stalls. + async fn acked( + &self, + token: WriteToken, + timeout: Duration, + bucket: &str, + key: &str, + metrics: &Metrics, + on_stall: &str, ) -> bool { let outcome: WriteWait = self.wait_cluster_applied(token, timeout).await; let retires_feed = outcome.retires_feed(self.consistency); @@ -823,10 +896,7 @@ impl WriteSync { } WriteWait::Stalled { waiting_on, lapsed } => { let who = node_names(&waiting_on); - warn!( - "write ack timed out for {bucket}/{key} ({who}); \ - answering 503, the write may be applied at the origin" - ); + warn!("write ack timed out for {bucket}/{key} ({who}); {on_stall}"); metrics.ack_timeout(); if !lapsed.is_empty() { // The lease tier *did* resolve this write, on a lapse, and the @@ -848,7 +918,7 @@ impl WriteSync { // `retire_through` retains the current head as an anchor and re-advertises the // compacted window; a dropped advertisement is carried by the next publish. if retires_feed { - self.feed.retire_through(token).await; + let _ = self.feed.retire_through(token).await; } guaranteed } @@ -898,136 +968,6 @@ impl WriteSync { .unwrap_or(false) } - /// Spawn the apply loop: peer events update the LIST index and invalidate - /// hot bodies. A peer write that describes nothing fences its key, and - /// `reconcile` resolves that fence from the origin. A gap synchronously - /// closes Groupnet's local serving gate, discards the index, and schedules - /// its rebuild. In `strong`, a small watcher reports the lease's monotone - /// lapse count to that same recovery driver. - /// - /// Takes `self` as an [`Arc`] because the lapse watcher holds a weak - /// reference back to this object, avoiding a task/handle cycle. - pub(crate) fn start_apply( - self: &Arc, - local: LocalCache, - state: Arc, - metrics: Arc, - reconcile: KeyReconcile, - ) { - let (frontier, view) = Frontier::new(); - let _ = self.view.set(view); - // Declare this node's feed writer before any capture can start, at - // its current position (zero for a feed with no writes yet). - state.register_native_writer(&self.own_position()); - if let Some(leases) = &self.leases { - let weak = Arc::downgrade(self); - let poll = lapse_poll(leases.config().duration); - tokio::spawn(async move { - loop { - tokio::time::sleep(poll).await; - let Some(sync) = weak.upgrade() else { return }; - let Some(recovery) = sync.recovery.get() else { - return; - }; - let count = sync.lease_view.as_ref().map_or(0, LeaseView::lapses); - if count > recovery.status().state.covered_lapses { - let _ = recovery.lease_lapse(count); - } - } - }); - } - // Bounded mode publishes no acks (that is its point at scale); both strong - // spellings do, and say so through their capability advertisement. - let ledger = self - .consistency - .acks() - .then(|| AckLedger::new(self.group.clone())); - let mut peers = PeerWrites::new(self.group.clone(), self.me.clone(), decode_event); - let weak = Arc::downgrade(self); - let lapse_view = self.lease_view.clone(); - tokio::spawn(async move { - while let Some(event) = peers.next().await { - let Some(sync) = weak.upgrade() else { return }; - match event { - PeerWrite::Sealed { peer, token } => { - state.note_native_seal(native_cut(&peer, token)); - sync.observe_seal(&peer, token); - frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } - } - PeerWrite::Renewed { - peer, - sealed, - epoch, - } => { - let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); - frontier.advance(&peer, start); - if let Some(ledger) = &ledger { - ledger.record(&peer, start).await; - } - } - PeerWrite::Wrote { - peer, - token, - key: event, - } => { - let cache_key = (event.bucket.clone(), event.key.clone()); - let mutation = local.fence_mutation(&cache_key).await; - let fenced = apply_feed_index(&state, &peer, token, &event); - drop(mutation); - // Fenced before the acknowledgement below: from then on this - // node answers the key from the origin until the reconciliation - // resolves it. - if let Some(fence) = fenced { - reconcile(&event.bucket, &event.key, fence); - } - // The index must move first: a warm body promoted after this hot - // eviction decodes suspect and is checked against the new entry - // before it can be served. Awaiting moka removal before the ack - // closes the stale-hot window without putting disk I/O on the - // feed frontier. - local.invalidate_hot(&cache_key).await; - frontier.advance(&peer, token); - if let Some(ledger) = &ledger { - ledger.record(&peer, token).await; - } - metrics.feed_applied(); - // The index has moved past any copy this node holds, so the warm - // copy can never serve again; left in place it would only crowd - // live bodies out of the disk budget until LRU found it. Disk I/O - // stays off the frontier: retire it after the acknowledgement. - let retire = local.clone(); - tokio::spawn(async move { retire.invalidate(&cache_key).await }); - } - PeerWrite::Gap { - peer, - missed_through, - } => { - warn!("write-feed gap from `{peer}`: distrusting bodies, resyncing index"); - sync.forget_crossing(&peer); - state.discard_for_gap(&native_cut(&peer, missed_through)); - // The public signal fences serving before returning; the - // recovery worker distrusts bodies and guards every origin - // scan page with that exact operation's permit. Frontier - // progress below means the old local state is unservable, - // not that the replacement index has materialized already. - if let Some(recovery) = sync.recovery.get() { - let lapses = lapse_view.as_ref().map_or(0, LeaseView::lapses); - let _ = recovery.feed_gap(lapses); - } - frontier.advance(&peer, missed_through); - if let Some(ledger) = &ledger { - ledger.record(&peer, missed_through).await; - } - metrics.feed_gap(); - } - } - } - }); - } - /// Wait (bounded by `timeout`) until every peer's currently-advertised /// feed head has been applied locally. Returns whether it fully caught /// up; freshness is bounded by one push/gossip hop (see module docs). diff --git a/src/sync/coherence/apply.rs b/src/sync/coherence/apply.rs new file mode 100644 index 0000000..ceedea5 --- /dev/null +++ b/src/sync/coherence/apply.rs @@ -0,0 +1,168 @@ +use std::sync::Arc; + +use groupnet::consistency::lease::LeaseView; +use groupnet::consistency::{AckLedger, Frontier, PeerWrite, PeerWrites, WriteToken}; +use groupnet::core::NodeId; +use tracing::warn; + +use crate::index::KeyIndex; +use crate::index::pending::UncertainKey; +use crate::metrics::Metrics; +use crate::sync::coherence::{KeyReconcile, WriteSync, apply_feed_index, lapse_poll, native_cut}; +use crate::sync::wire::decode_event; +use crate::tier::LocalCache; + +impl WriteSync { + /// Spawn the apply loop: peer events update the LIST index and invalidate + /// hot bodies. A peer write that describes nothing fences its key, and + /// `reconcile` resolves that fence from the origin; a pending one fences it + /// until its writer settles it, or its next life hands it to `reconcile`. A gap synchronously + /// closes Groupnet's local serving gate, discards the index, and schedules + /// its rebuild. In `strong`, a small watcher reports the lease's monotone + /// lapse count to that same recovery driver. + /// + /// Takes `self` as an [`Arc`] because the lapse watcher holds a weak + /// reference back to this object, avoiding a task/handle cycle. + pub(crate) fn start_apply( + self: &Arc, + local: LocalCache, + state: Arc, + metrics: Arc, + reconcile: KeyReconcile, + ) { + let (frontier, view) = Frontier::new(); + let _ = self.view.set(view); + // Declare this node's feed writer before any capture can start, at + // its current position (zero for a feed with no writes yet). + state.register_native_writer(&self.own_position()); + if let Some(leases) = &self.leases { + let weak = Arc::downgrade(self); + let poll = lapse_poll(leases.config().duration); + tokio::spawn(async move { + loop { + tokio::time::sleep(poll).await; + let Some(sync) = weak.upgrade() else { return }; + let Some(recovery) = sync.recovery.get() else { + return; + }; + let count = sync.lease_view.as_ref().map_or(0, LeaseView::lapses); + if count > recovery.status().state.covered_lapses { + let _ = recovery.lease_lapse(count); + } + } + }); + } + // Bounded mode publishes no acks (that is its point at scale); both strong + // spellings do, and say so through their capability advertisement. + let ledger = self + .consistency + .acks() + .then(|| AckLedger::new(self.group.clone())); + let mut peers = PeerWrites::new(self.group.clone(), self.me.clone(), decode_event); + let weak = Arc::downgrade(self); + let lapse_view = self.lease_view.clone(); + tokio::spawn(async move { + while let Some(event) = peers.next().await { + let Some(sync) = weak.upgrade() else { return }; + match event { + PeerWrite::Sealed { peer, token } => { + state.note_native_seal(native_cut(&peer, token)); + let ended = state.end_pending_life( + peer.as_str().as_bytes(), + token.epoch.saturating_add(1), + ); + reconcile_all(&reconcile, ended); + sync.observe_seal(&peer, token); + frontier.advance(&peer, token); + acknowledge(ledger.as_ref(), &peer, token).await; + } + PeerWrite::Renewed { + peer, + sealed, + epoch, + } => { + let start = sync.observe_renewal(&state, &peer, sealed, epoch, &metrics); + reconcile_all( + &reconcile, + state.end_pending_life(peer.as_str().as_bytes(), epoch), + ); + frontier.advance(&peer, start); + acknowledge(ledger.as_ref(), &peer, start).await; + } + PeerWrite::Wrote { + peer, + token, + key: event, + } => { + let cache_key = (event.bucket.clone(), event.key.clone()); + let mutation = local.fence_mutation(&cache_key).await; + let fenced = apply_feed_index(&state, &peer, token, &event); + drop(mutation); + // Fenced before the acknowledgement below: from then on this + // node answers these keys from the origin until each + // reconciliation resolves its key. + reconcile_all(&reconcile, fenced); + // The index must move first: a warm body promoted after this hot + // eviction decodes suspect and is checked against the new entry + // before it can be served. Awaiting moka removal before the ack + // closes the stale-hot window without putting disk I/O on the + // feed frontier. + local.invalidate_hot(&cache_key).await; + frontier.advance(&peer, token); + acknowledge(ledger.as_ref(), &peer, token).await; + metrics.feed_applied(); + // The index has moved past any copy this node holds, so the warm + // copy can never serve again; left in place it would only crowd + // live bodies out of the disk budget until LRU found it. Disk I/O + // stays off the frontier: retire it after the acknowledgement. + let retire = local.clone(); + tokio::spawn(async move { retire.invalidate(&cache_key).await }); + } + PeerWrite::Gap { + peer, + missed_through, + } => { + warn!("write-feed gap from `{peer}`: distrusting bodies, resyncing index"); + sync.forget_crossing(&peer); + state.discard_for_gap(&native_cut(&peer, missed_through)); + // A gap within one life keeps that life's pending writes: + // the floor of its next event settles them. + reconcile_all( + &reconcile, + state.end_pending_life(peer.as_str().as_bytes(), missed_through.epoch), + ); + // The public signal fences serving before returning; the + // recovery worker distrusts bodies and guards every origin + // scan page with that exact operation's permit. Frontier + // progress below means the old local state is unservable, + // not that the replacement index has materialized already. + if let Some(recovery) = sync.recovery.get() { + let lapses = lapse_view.as_ref().map_or(0, LeaseView::lapses); + let _ = recovery.feed_gap(lapses); + } + frontier.advance(&peer, missed_through); + acknowledge(ledger.as_ref(), &peer, missed_through).await; + metrics.feed_gap(); + } + } + } + }); + } +} + +/// Start the origin reconciliation of each key left uncertain. +fn reconcile_all(reconcile: &KeyReconcile, keys: Vec) { + for (bucket, key, token) in keys { + reconcile(&bucket, &key, token); + } +} + +/// Advertise that this node applied `peer`'s feed through `token`. A refused +/// advertisement keeps the advance locally, and the next advancing record carries it. +async fn acknowledge(ledger: Option<&AckLedger>, peer: &NodeId, token: WriteToken) { + if let Some(ledger) = ledger + && let Err(error) = ledger.record(peer, token).await + { + warn!("could not advertise the ack of `{peer}` through {token:?}: {error}"); + } +} diff --git a/src/sync/coherence/fence.rs b/src/sync/coherence/fence.rs new file mode 100644 index 0000000..1f8915f --- /dev/null +++ b/src/sync/coherence/fence.rs @@ -0,0 +1,168 @@ +//! The pre-write fence. In `strong`, a write is announced to every peer, and the +//! announcement acknowledged (or the peer's lease lapsed), before the write reaches the +//! origin. A writer that crashes after the origin applied the write therefore leaves +//! every peer answering the key from the origin (see [`crate::index::pending`]). + +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, SystemTime}; + +use groupnet::consistency::{PublishError, WriteToken}; +use groupnet::core::volatile_bootstrap::journal::NativeCut; + +use crate::index::KeyIndex; +use crate::metrics::Metrics; +use crate::sync::coherence::{Consistency, WriteReceipt, WriteSync}; +use crate::sync::wire::{IndexEvent, IndexOp, to_micros}; + +type Open = Arc>>; + +fn lock(open: &Open) -> MutexGuard<'_, BTreeSet> { + open.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// The feed sequences of this node's pending writes not settled yet. +#[derive(Default)] +pub(super) struct OpenWrites(Open); + +/// A write announced to every peer before its origin call. The event reporting its +/// outcome settles it. Dropped unsettled (its task died), it stops holding back the +/// floor the writer's next event carries. +pub(crate) struct PendingWrite { + seq: u64, + open: Open, +} + +impl Drop for PendingWrite { + fn drop(&mut self) { + lock(&self.open).remove(&self.seq); + } +} + +/// The fence did not reach every peer in time. Its announcements are abandoned and +/// the write must not reach the origin. +#[derive(Debug)] +pub(crate) struct FenceStalled; + +impl WriteSync { + /// Stamp an own event with the pending write it settles and the oldest one still + /// open, and publish it. Runs under the index lock that orders every own event; + /// `opens` records the event as a new pending write. + pub(super) fn publish_own( + &self, + mut event: IndexEvent, + settles: Option<&PendingWrite>, + opens: bool, + ) -> ( + impl Future> + Send + use<'_>, + NativeCut, + ) { + let mut open = lock(&self.open_writes.0); + if let Some(pending) = settles { + open.remove(&pending.seq); + event.settles = Some(pending.seq); + } + event.floor = open.first().copied(); + let publishing = self.feed.publish(&event); + let position = self.own_position(); + if opens { + open.insert(position.sequence); + } + (publishing, position) + } + + /// Announce a write of each of `keys` and wait until every peer that could serve + /// them holds the announcement or has lapsed. Only then may the write reach the + /// origin, and each returned [`PendingWrite`] is settled by the event that reports + /// its key's outcome. Only `strong` fences; other modes get `None` for each key. + /// + /// # Errors + /// [`FenceStalled`] when the wait stalled. The announcements are then abandoned. + pub(crate) async fn fence( + &self, + state: &KeyIndex, + bucket: &str, + keys: &[String], + timeout: Duration, + metrics: &Metrics, + ) -> Result>, FenceStalled> { + if self.consistency != Consistency::Strong { + return Ok(keys.iter().map(|_| None).collect()); + } + let mut pending = Vec::with_capacity(keys.len()); + let mut newest = None; + for key in keys { + let (write, receipt) = self.index_pending(state, bucket, key, metrics).await; + pending.push(write); + newest = Some(receipt.token); + } + let Some(token) = newest else { + return Ok(Vec::new()); + }; + let label = match keys { + [key] => key.as_str(), + _ => "", + }; + let on_stall = "answering 503, the write was not sent to the origin"; + if self + .acked(token, timeout, bucket, label, metrics, on_stall) + .await + { + return Ok(pending.into_iter().map(Some).collect()); + } + for (key, write) in keys.iter().zip(pending) { + self.index_abandoned(state, bucket, key, write, metrics) + .await; + } + Err(FenceStalled) + } + + async fn index_pending( + &self, + state: &KeyIndex, + bucket: &str, + key: &str, + metrics: &Metrics, + ) -> (PendingWrite, WriteReceipt) { + let event = IndexEvent { + op: IndexOp::Pending, + bucket: bucket.to_owned(), + key: key.to_owned(), + ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, + }; + let (publishing, seq) = state.note_own_native(|| { + let (publishing, position) = self.publish_own(event, None, true); + ((publishing, position.sequence), position) + }); + let write = PendingWrite { + seq, + open: Arc::clone(&self.open_writes.0), + }; + (write, self.advertised(publishing, metrics).await) + } + + /// Settle a pending write the origin refused, or that was never sent: the key is + /// as it was, so peers stop fencing it without waiting for anyone. + pub(crate) async fn index_abandoned( + &self, + state: &KeyIndex, + bucket: &str, + key: &str, + write: PendingWrite, + metrics: &Metrics, + ) { + let event = IndexEvent { + op: IndexOp::Abandoned, + bucket: bucket.to_owned(), + key: key.to_owned(), + ts_us: to_micros(SystemTime::now()), + settles: None, + floor: None, + }; + let publishing = state.note_own_native(|| self.publish_own(event, Some(&write), false)); + drop(write); + self.advertised(publishing, metrics).await; + } +} diff --git a/src/sync/config.rs b/src/sync/config.rs index 8122220..3d67171 100644 --- a/src/sync/config.rs +++ b/src/sync/config.rs @@ -1,10 +1,11 @@ +use std::num::NonZeroU32; use std::sync::{Arc, OnceLock}; use std::time::Duration; -use groupnet::consistency::LeaseConfig; +use groupnet::consistency::lease::LeaseConfig; use groupnet::core::{Config, NodeId}; use groupnet::runtime::{Group, NamedSeeds, Node, SeedEvent, SystemResolver}; -use groupnet::transport::Transport; +use groupnet::transport::link::{BoundLink, LinkConfig, LinkProvider}; use groupnet::transport::udp::UdpTransport; use tracing::{info, warn}; @@ -13,7 +14,7 @@ use crate::sync::readiness::UnmetSeeds; /// Attempts to resolve a gossip seed's DNS name before it is reported /// unresolvable — a `StatefulSet` peer's record can lag its own startup. -const SEED_RESOLVE_ATTEMPTS: u32 = 30; +const SEED_RESOLVE_ATTEMPTS: NonZeroU32 = NonZeroU32::new(30).unwrap(); /// The wait between those first attempts. const SEED_RETRY: Duration = Duration::from_secs(1); @@ -28,9 +29,9 @@ const SEED_REFRESH: Duration = Duration::from_secs(15); /// s3cache's startup window and refresh cadence. fn seed_resolution() -> NamedSeeds { NamedSeeds::new(SystemResolver) - .retry_interval(SEED_RETRY) - .startup_attempts(SEED_RESOLVE_ATTEMPTS) - .refresh_interval(SEED_REFRESH) + .with_retry_interval(SEED_RETRY) + .with_startup_attempts(SEED_RESOLVE_ATTEMPTS) + .with_refresh_interval(SEED_REFRESH) } /// Logs what the seed resolver did and releases readiness for a seed that @@ -63,9 +64,9 @@ pub struct SyncConfig { /// The address peers should reach this node on; the bound address when /// `None` (`S3CACHE_GOSSIP_ADVERTISE`). pub advertise: Option, - /// Statically-addressed peers as `(node id, host:port)`. Every other peer - /// resolves itself through gossiped advertisements, so only seeds need - /// static addressing (`S3CACHE_GOSSIP_SEEDS`). + /// Explicit routing neighbors as `(node id, host:port)`. DNS resolution and + /// gossiped advertisements refresh their addresses, but do not admit additional + /// neighbors (`S3CACHE_GOSSIP_SEEDS`). pub seeds: Vec<(String, String)>, /// This node's identity in the cluster (the pod name, in the chart). pub node_id: String, @@ -127,12 +128,15 @@ impl WriteSync { let lease = lease_config(cfg.lease_ms); let advertise = cfg .advertise - .or_else(|| transport.local_addr().ok().map(|addr| addr.to_string())); + .unwrap_or_else(|| transport.local_addr().to_string()); + // A link admits each neighbor once; a repeated seed id names the same peer. let seeds: Vec = cfg .seeds .iter() .filter(|(id, _)| *id != cfg.node_id) // a pod seeding itself (uniform config) is a no-op .map(|(id, _)| NodeId::new(id.as_str())) + .collect::>() + .into_iter() .collect(); // Seeds resolve off the startup path (DNS for a just-starting peer may // lag, and a slow resolver must not delay serving), inside groupnet: @@ -141,64 +145,64 @@ impl WriteSync { // the first `Unresolved` comes a whole startup window later. let unmet = Arc::new(OnceLock::new()); let observed = Arc::clone(&unmet); - let mut resolution = resolution.on_event(move |event| observe_seed(event, &observed)); + let mut resolution = resolution.with_observer(move |event| observe_seed(event, &observed)); for (id, addr) in &cfg.seeds { - resolution = resolution.seed(NodeId::new(id.as_str()), addr.as_str()); + resolution = resolution.with_seed(NodeId::new(id.as_str()), addr.as_str()); } - let mut builder = Node::builder(me.clone(), transport) + let node = Node::builder(me.clone()) + .link(BoundLink::new(transport, LinkConfig::new(seeds.clone()))) .config(gossip_config(cfg.lease_ms)) - .named_seeds(resolution); - if let Some(advertise) = advertise { - builder = builder.advertise_addr(advertise); - } - let node = builder.spawn(); + .named_seeds(resolution) + .advertise_addr(advertise); + let node = match node.start().await { + Ok(node) => node, + Err(error) => { + warn!("gossip disabled: cannot start managed network: {error}"); + return None; + } + }; let group = node.join_group("s3cache"); let (bind, node_id, mode) = (&cfg.bind, &cfg.node_id, cfg.consistency.label()); let lease_ms = cfg.lease_ms; info!( "gossip coherence bound on `{bind}` as `{node_id}` (consistency: {mode}, lease: {lease_ms}ms)" ); - let sync = WriteSync::attach(group, me, cfg.consistency, lease, Some(Box::new(node))); + let sync = WriteSync::attach(group, me, cfg.consistency, lease, Some(node)); let _ = unmet.set(sync.expect_seeds(seeds)); Some(sync) } - /// Join the cluster group as `node_id` over an already-built groupnet - /// `transport`, seeded with `seeds`, with exactly the membership tuning and - /// coherence lease [`WriteSync::new`] runs. `new` is this over UDP plus seed - /// address resolution; any other transport (the in-memory one, for a whole - /// fleet in one process) resolves its own peers. + /// Join the cluster group as `node_id` over a managed protocol `link`, with + /// exactly the membership tuning and coherence lease [`WriteSync::new`] runs. + /// The link explicitly admits its routing neighbors; `seeds` additionally + /// records the peers readiness must await. /// - /// # Panics - /// Outside a Tokio runtime: the node spawns its tasks here. - pub fn over_transport( - transport: T, + /// # Errors + /// Returns an error if the link cannot bind or the managed network cannot start. + pub async fn over_link( + link: impl LinkProvider, node_id: &str, seeds: &[&str], consistency: Consistency, lease_ms: u64, - ) -> Self { + ) -> std::io::Result { let me = NodeId::new(node_id); - let mut builder = Node::builder(me.clone(), transport).config(gossip_config(lease_ms)); + let mut builder = Node::builder(me.clone()) + .link(link) + .config(gossip_config(lease_ms)); for seed in seeds.iter().filter(|seed| **seed != node_id) { builder = builder.seed(NodeId::new(*seed)); } - let node = builder.spawn(); + let node = builder.start().await?; let group = node.join_group("s3cache"); - let sync = WriteSync::attach( - group, - me, - consistency, - lease_config(lease_ms), - Some(Box::new(node)), - ); + let sync = WriteSync::attach(group, me, consistency, lease_config(lease_ms), Some(node)); sync.expect_seeds( seeds .iter() .filter(|seed| **seed != node_id) .map(|seed| NodeId::new(*seed)), ); - sync + Ok(sync) } /// The gossip group this node's feed, leases and membership ride on, for @@ -280,3 +284,43 @@ pub(super) fn parse_seeds(raw: &str) -> Vec<(String, String)> { }) .collect() } + +/// Why [`CachingProxy::with_recovery_config`](crate::cache::proxy::CachingProxy::with_recovery_config) +/// refused a recovery configuration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecoveryConfigRejected { + /// Groupnet's own bounds refused it. + Invalid(groupnet::core::volatile_recovery::RecoveryConfigError), + /// The settle window is shorter than two gossip rounds of this node, so a + /// renewal could still be in flight when the window closes. + SettleBelowGossip { + /// The configured settle window. + settle_ms: u64, + /// Two of this node's gossip rounds. + floor_ms: u64, + }, +} + +impl std::fmt::Display for RecoveryConfigRejected { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Invalid(error) => error.fmt(f), + Self::SettleBelowGossip { + settle_ms, + floor_ms, + } => write!( + f, + "recovery settle_ms {settle_ms} is below two gossip rounds ({floor_ms} ms)" + ), + } + } +} + +impl std::error::Error for RecoveryConfigRejected { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Invalid(error) => Some(error), + Self::SettleBelowGossip { .. } => None, + } + } +} diff --git a/src/sync/stop.rs b/src/sync/stop.rs index f14eb5a..0c017af 100644 --- a/src/sync/stop.rs +++ b/src/sync/stop.rs @@ -20,7 +20,7 @@ use std::time::Duration; use groupnet::consistency::volatile_recovery::{Mark, Renewal}; -use groupnet::consistency::{WriteToken, applied_by_selected}; +use groupnet::consistency::{AckWait, WriteToken, applied_by_selected}; use groupnet::core::NodeId; use tracing::{info, warn}; @@ -85,10 +85,20 @@ impl WriteSync { } } + /// Close the managed gossip network after the mutation drain and feed seal. + /// This cancels address resolution and releases every owned link endpoint. + pub async fn close(&self) { + if let Some(node) = &self.node { + node.close().await; + } + } + /// Advertise this life's feed epoch before any write, so every peer learns of the /// restart now: a sealed previous life renews, an unsealed one gaps at once. pub(crate) async fn announce(&self) { - self.feed.republish().await; + if let Err(error) = self.feed.republish().await { + warn!("could not announce this life's write feed: {error}"); + } } /// Seal this life's write feed and wait up to `wait` for the peers a write waits @@ -97,18 +107,29 @@ impl WriteSync { /// Call it only once nothing can publish again: the HTTP drain has completed and /// every mutation tail has finished. The seal promises the peers that this life wrote /// nothing after it, which is what lets them skip the restart remediation; a - /// write published afterwards panics in the feed rather than break that promise. + /// write published afterwards panics rather than break that promise. pub async fn seal(&self, wait: Duration) -> SealOutcome { - let token = self.feed.seal().await; + let token = match self.feed.seal().await { + Ok(token) => token, + // Sealed locally either way; a peer that never sees the sealed frame gaps + // after the restart, and the acknowledgement wait below reports it. + Err(unadvertised) => { + warn!("{unadvertised}"); + unadvertised.token + } + }; let observed = self.consistency.acks() && applied_by_selected( &self.group, - &self.me, - token, + AckWait { + writer: &self.me, + token, + timeout: wait, + }, |node| waits_on(&self.group, node), - wait, ) - .await; + .await + .is_complete(); if observed { info!( epoch = token.epoch, diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 73001ec..03d4dd9 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -1,15 +1,20 @@ +use std::num::NonZeroU32; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; +use groupnet::consistency::CAP_ACKS; +use groupnet::consistency::lease::{CAP_LEASE, LeaseConfig}; use groupnet::consistency::volatile_recovery::{RecoveryStage, RecoveryStatus}; -use groupnet::consistency::{CAP_ACKS, CAP_LEASE, LeaseConfig}; use groupnet::core::volatile_recovery::RecoveryState; use groupnet::core::{Config, NodeId, Status}; -use groupnet::runtime::{Group, NamedSeeds, Node, ResolveFuture, SeedResolver, SystemResolver}; -use groupnet::transport::mem::{MemTransport, Network}; +use groupnet::runtime::{NamedSeeds, ResolveFuture, SeedResolver, SystemResolver}; +use groupnet::transport::mem::Network; use s3s::dto::GetObjectOutput; -use crate::index::{KeyIndex, ObjEntry, standard_class}; +use crate::index::{ + AuthoritativeKeyState, KeyIndex, ObjEntry, resolve_uncertain_key, standard_class, + uncertain_key_is_current, +}; use crate::metrics::Metrics; use crate::sync::coherence::{ CAP_BOUNDED, Consistency, DEFAULT_LEASE_MS, KeyReconcile, WriteReceipt, WriteSync, WriteWait, @@ -21,6 +26,9 @@ use crate::sync::wire::{ }; use crate::tier::{CachedObject, TieredCache, open_warm}; +mod fixtures; +use fixtures::{attach, spawn_neighbors, spawn_node, wired_pair, wired_pair_named}; + type Index = Arc; /// A lease short enough to watch lapse inside a test, and still comfortably inside @@ -33,7 +41,7 @@ fn test_lease() -> LeaseConfig { /// Index `entry` as this writer's own put into a scratch index and advertise it. async fn own_put(sync: &WriteSync, key: &str, entry: ObjEntry, metrics: &Metrics) -> WriteReceipt { - sync.index_put(&KeyIndex::default(), "bkt", key, entry, metrics) + sync.index_put(&KeyIndex::default(), "bkt", key, entry, None, metrics) .1 .await } @@ -69,12 +77,11 @@ fn a_reopened_recovery_generation_cannot_authorize_an_old_local_answer() { /// reap-bounded behaviour is the same shape production gets, only faster. fn brisk() -> Config { Config { - gossip_interval_ms: 10, + gossip_interval_ms: 25, probe_interval_ms: 20, probe_timeout_ms: 10, suspect_timeout_ms: 50, dead_timeout_ms: 300, - anti_entropy_interval_ms: 25, ..Config::default() } } @@ -112,22 +119,6 @@ fn only_acknowledged_strong_waits_retire_feed_history() { assert!(!WriteWait::Lapsed(Vec::new()).retires_feed(Consistency::Bounded)); } -fn spawn_node(net: &Network, id: &str, peer: &str) -> (NodeId, Node, Group) { - let me = NodeId::new(id); - let node = Node::builder(me.clone(), net.endpoint(me.clone())) - .seed(NodeId::new(peer)) - .config(brisk()) - .spawn(); - let group = node.join_group("s3cache"); - (me, node, group) -} - -/// A [`WriteSync`] on `group`, leased at [`TEST_LEASE`] unless the mode says -/// otherwise. -fn attach(group: Group, me: NodeId, consistency: Consistency) -> WriteSync { - WriteSync::attach(group, me, consistency, test_lease(), None) -} - fn cached(body: &'static [u8]) -> Arc { Arc::new(CachedObject::from_get( &GetObjectOutput::default(), @@ -154,33 +145,6 @@ fn indexed(state: &Index, bucket: &str, key: &str) -> Option { .cloned() } -/// A fully-wired pair: node A publishes, node B applies into `state` and -/// its local cache. Returns A's publisher, B's sync, and B's state and cache. -fn wired_pair(net: &Network) -> (WriteSync, Arc, Index, TieredCache) { - wired_pair_named(net, ("sync-a", "sync-b"), Consistency::Strong) -} - -/// [`wired_pair`] with the gossip identities and the mode spelled out — unique ids -/// per test, so parallel tests never share one. -/// -/// B comes back as an [`Arc`]: it is the node whose apply loop runs, and -/// [`WriteSync::start_apply`] hands the lapse watch a [`Weak`] back to it. -fn wired_pair_named( - net: &Network, - ids: (&str, &str), - consistency: Consistency, -) -> (WriteSync, Arc, Index, TieredCache) { - let (a_id, _a_node, a_group) = spawn_node(net, ids.0, ids.1); - let (b_id, _b_node, b_group) = spawn_node(net, ids.1, ids.0); - let metrics = Arc::new(Metrics::default()); - let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); - let state = Arc::new(KeyIndex::default()); - let sync_b = Arc::new(attach(b_group, b_id, consistency)); - sync_b.start_apply(cache.local(), state.clone(), metrics, no_reconcile()); - let sync_a = attach(a_group, a_id, consistency); - (sync_a, sync_b, state, cache) -} - async fn eventually(mut cond: impl FnMut() -> bool, what: &str) { for _ in 0..300 { if cond() { @@ -240,7 +204,7 @@ fn env_spellings_parse_into_a_config() { const TEST_SEED_CADENCE: Duration = Duration::from_millis(20); /// Failed lookups before the test's unresolvable seed is reported. -const TEST_STARTUP_ATTEMPTS: u32 = 3; +const TEST_STARTUP_ATTEMPTS: NonZeroU32 = NonZeroU32::new(3).unwrap(); /// How long a loopback gossip pair may take to (re)meet. const TEST_REJOIN: Duration = Duration::from_secs(20); @@ -301,9 +265,9 @@ async fn within(limit: Duration, what: &str, mut cond: impl FnMut() -> bool) { #[tokio::test(flavor = "multi_thread")] async fn an_unresolvable_seed_releases_readiness_after_its_window() { let resolution = NamedSeeds::new(MovableSeed::default()) - .retry_interval(TEST_SEED_CADENCE) - .startup_attempts(TEST_STARTUP_ATTEMPTS) - .refresh_interval(TEST_SEED_CADENCE); + .with_retry_interval(TEST_SEED_CADENCE) + .with_startup_attempts(TEST_STARTUP_ATTEMPTS) + .with_refresh_interval(TEST_SEED_CADENCE); let sync = WriteSync::bind( seed_config("seed-lone", loopback(), &[("seed-ghost", "ghost:1")]), resolution, @@ -321,22 +285,23 @@ async fn an_unresolvable_seed_releases_readiness_after_its_window() { } /// A seed that comes back at a new address is relearned by re-resolution, and -/// this node reaches the new life — which, seeding nobody, would otherwise -/// never hear from it. +/// this node reaches the new life. Both endpoints explicitly admit each other; +/// discovering an address does not admit a new routing neighbor. #[tokio::test(flavor = "multi_thread")] async fn a_moved_seed_is_relearned_and_rejoins() { let (a, b_first, b_moved) = (loopback(), loopback(), loopback()); let name_b = MovableSeed::default(); name_b.point_at(Some(b_first)); let resolution = NamedSeeds::new(name_b.clone()) - .retry_interval(TEST_SEED_CADENCE) - .refresh_interval(TEST_SEED_CADENCE); + .with_retry_interval(TEST_SEED_CADENCE) + .with_refresh_interval(TEST_SEED_CADENCE); let node_a = WriteSync::bind(seed_config("seed-a", a, &[("seed-b", "b:1")]), resolution) .await .expect("binds loopback"); + let address_a = a.to_string(); let peer = |bind| { WriteSync::bind( - seed_config("seed-b", bind, &[]), + seed_config("seed-b", bind, &[("seed-a", address_a.as_str())]), NamedSeeds::new(SystemResolver), ) }; @@ -393,11 +358,12 @@ fn only_a_mode_that_grants_leases_advertises_that_it_does() { #[tokio::test] async fn the_ack_wait_covers_advertisers_and_unknowns_but_not_declared_bounded_peers() { let net = Network::new(); - let (_a_id, _a_node, a_group) = spawn_node(&net, "cap-a", "cap-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "cap-b", "cap-a"); - let (c_id, _c_node, c_group) = spawn_node(&net, "cap-c", "cap-a"); + let (_a_id, _a_node, a_group) = + spawn_neighbors(&net, "cap-a", &["cap-b", "cap-c", "cap-d"]).await; + let (b_id, _b_node, b_group) = spawn_node(&net, "cap-b", "cap-a").await; + let (c_id, _c_node, c_group) = spawn_node(&net, "cap-c", "cap-a").await; // D runs, joins, and never advertises — a node from before this change. - let (d_id, _d_node, _d_group) = spawn_node(&net, "cap-d", "cap-a"); + let (d_id, _d_node, _d_group) = spawn_node(&net, "cap-d", "cap-a").await; b_group .advertise_capabilities([CAP_ACKS]) .expect("B advertises the ack tier"); @@ -427,12 +393,13 @@ async fn the_ack_wait_covers_advertisers_and_unknowns_but_not_declared_bounded_p #[tokio::test] async fn the_transition_wait_covers_exactly_what_the_lease_cannot() { let net = Network::new(); - let (_a_id, _a_node, a_group) = spawn_node(&net, "mix-a", "mix-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "mix-b", "mix-a"); - let (c_id, _c_node, c_group) = spawn_node(&net, "mix-c", "mix-a"); + let (_a_id, _a_node, a_group) = + spawn_neighbors(&net, "mix-a", &["mix-b", "mix-c", "mix-d", "mix-e"]).await; + let (b_id, _b_node, b_group) = spawn_node(&net, "mix-b", "mix-a").await; + let (c_id, _c_node, c_group) = spawn_node(&net, "mix-c", "mix-a").await; // D runs, joins, and never advertises — a node from before any of this. - let (d_id, _d_node, _d_group) = spawn_node(&net, "mix-d", "mix-a"); - let (e_id, _e_node, e_group) = spawn_node(&net, "mix-e", "mix-a"); + let (d_id, _d_node, _d_group) = spawn_node(&net, "mix-d", "mix-a").await; + let (e_id, _e_node, e_group) = spawn_node(&net, "mix-e", "mix-a").await; b_group .advertise_capabilities(Consistency::StrongAcks.capabilities().iter().copied()) .expect("B pins the ack tier"); @@ -474,6 +441,8 @@ fn event_codec_round_trips_and_rejects_garbage() { bucket: "bucket-1".to_owned(), key: "a/b weird\0key".to_owned(), ts_us: 1_700_000_000_000_000, + settles: Some(7), + floor: Some(5), }; let encoded = encode_event(&event); assert_eq!( @@ -528,7 +497,7 @@ fn local_stamps_are_truncated_to_the_wire_precision() { #[tokio::test] async fn peer_events_fold_into_index_and_invalidate() { let net = Network::new(); - let (sync_a, _sync_b, state, cache) = wired_pair(&net); + let (sync_a, _sync_b, state, cache) = wired_pair(&net).await; let metrics = Metrics::default(); // B holds a soon-stale body copy. @@ -570,6 +539,7 @@ async fn peer_events_fold_into_index_and_invalidate() { "bkt", "obj", SystemTime::now(), + None, &metrics, ) .await; @@ -588,8 +558,8 @@ async fn peer_events_fold_into_index_and_invalidate() { #[tokio::test] async fn peer_delete_retires_the_warm_copy() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "warm-a", "warm-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "warm-b", "warm-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "warm-a", "warm-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "warm-b", "warm-a").await; let metrics = Arc::new(Metrics::default()); let dir = std::env::temp_dir().join(format!("s3cache-peer-warm-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); @@ -621,6 +591,7 @@ async fn peer_delete_retires_the_warm_copy() { "bkt", "gone", SystemTime::now(), + None, &metrics, ) .await; @@ -646,8 +617,8 @@ async fn peer_delete_retires_the_warm_copy() { #[tokio::test] async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "unknown-a", "unknown-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "unknown-b", "unknown-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "unknown-a", "unknown-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "unknown-b", "unknown-a").await; let metrics = Arc::new(Metrics::default()); let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); let state = Arc::new(KeyIndex::default()); @@ -676,7 +647,7 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { let ckey = ("bkt".to_owned(), "ptr".to_owned()); cache.insert(ckey.clone(), cached(b"old")).await; - let (_, publishing) = sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", &metrics); + let (_, publishing) = sync_a.index_unknown(&KeyIndex::default(), "bkt", "ptr", None, &metrics); let unknown = publishing.await; assert!( sync_b @@ -708,12 +679,200 @@ async fn an_undescribed_peer_write_fences_the_key_before_it_is_acknowledged() { assert_eq!(indexed_size(&state, "bkt", "ptr"), Some(5)); } +fn fenced(state: &Index, key: &str) -> bool { + state.read().unwrap().key_fenced("bkt", key) +} + +/// Until `writer` waits on `reader`'s serve-lease and not on its acks alone, a fence +/// could resolve without the reader in it. +async fn adopted(writer: &WriteSync, reader: &NodeId) { + eventually( + || { + writer.lease_holders() == std::slice::from_ref(reader) + && !waits_on_unleased(&writer.group, reader) + }, + "the writer to adopt the reader's serve-lease and its lease advertisement", + ) + .await; +} + +/// The pre-write fence: once it returns, the peer answers the key from the origin, and +/// the event reporting that write's outcome is what lifts it, for that write alone. +#[tokio::test] +async fn a_fenced_write_holds_the_peer_off_the_key_until_its_outcome_arrives() { + let net = Network::new(); + let (sync_a, sync_b, state, _cache) = + wired_pair_named(&net, ("fence-a", "fence-b"), Consistency::Strong).await; + adopted(&sync_a, &NodeId::new("fence-b")).await; + let metrics = Metrics::default(); + let own = KeyIndex::default(); + let first = own_put(&sync_a, "k", written(4), &metrics).await; + assert!( + sync_b + .reached_token(&first.header, Duration::from_secs(5)) + .await + ); + + let keys = ["k".to_owned(), "refused".to_owned()]; + let mut pending = sync_a + .fence(&own, "bkt", &keys, Duration::from_secs(5), &metrics) + .await + .expect("the peer acknowledges the fence"); + assert!( + fenced(&state, "k") && fenced(&state, "refused"), + "the peer holds the fence before the origin is called" + ); + let refused = pending.pop().flatten().expect("strong announces each key"); + let k = pending.pop().flatten().expect("strong announces each key"); + + let (_, publishing) = sync_a.index_put(&own, "bkt", "k", written(5), Some(&k), &metrics); + let outcome = publishing.await; + assert!( + sync_b + .reached_token(&outcome.header, Duration::from_secs(5)) + .await + ); + assert!(!fenced(&state, "k")); + assert_eq!(indexed_size(&state, "bkt", "k"), Some(5)); + assert!( + fenced(&state, "refused"), + "an outcome lifts only the write it settles" + ); + + sync_a + .index_abandoned(&own, "bkt", "refused", refused, &metrics) + .await; + eventually( + || !fenced(&state, "refused"), + "the abandonment lifts the fence", + ) + .await; + assert_eq!(indexed_size(&state, "bkt", "refused"), None); +} + +/// A writer that dies after its fence, before its origin call or after it, cannot say +/// what the origin holds. Its peer keeps the keys origin-only, and the writer's next +/// life hands them to the origin reconciliation. A write it never fenced never reached +/// the origin, so it leaves nothing to reconcile. +#[tokio::test] +async fn a_writer_that_dies_after_its_fence_leaves_the_keys_to_the_origin() { + let net = Network::new(); + let (a_id, a_node, a_group) = spawn_node(&net, "died-a", "died-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "died-b", "died-a").await; + let metrics = Arc::new(Metrics::default()); + let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); + let state = Arc::new(KeyIndex::default()); + let reconciling = Arc::new(std::sync::Mutex::new(Vec::new())); + let sync_b = Arc::new(attach(b_group, b_id.clone(), Consistency::Strong)); + let handed = Arc::clone(&reconciling); + sync_b.start_apply( + cache.local(), + state.clone(), + metrics.clone(), + Arc::new(move |bucket: &str, key: &str, token: u64| { + handed + .lock() + .unwrap() + .push((bucket.to_owned(), key.to_owned(), token)); + }), + ); + let sync_a = attach(a_group, a_id, Consistency::Strong); + adopted(&sync_a, &b_id).await; + + // `unsent` dies before its origin call; `applied` after the origin applied it. + let keys = ["unsent".to_owned(), "applied".to_owned()]; + let pending = sync_a + .fence( + &KeyIndex::default(), + "bkt", + &keys, + Duration::from_secs(5), + &metrics, + ) + .await + .expect("the peer acknowledges the fence"); + // The crash: no outcome and no seal. + drop(pending); + drop(sync_a); + a_node.close().await; + drop(a_node); + assert!(fenced(&state, "unsent") && fenced(&state, "applied")); + assert!(reconciling.lock().unwrap().is_empty()); + + // The pod returns under the same id as a new life, and writes another key. + let (a_id, _a_node, a_group) = spawn_node(&net, "died-a", "died-b").await; + let sync_a = attach(a_group, a_id, Consistency::Strong); + own_put(&sync_a, "other", written(1), &metrics).await; + eventually( + || reconciling.lock().unwrap().len() == 2, + "the new life hands both keys to the origin reconciliation", + ) + .await; + let handed: Vec<(String, String, u64)> = reconciling.lock().unwrap().clone(); + assert!( + handed.iter().all(|(_, key, _)| key != "other"), + "an unfenced write leaves nothing to reconcile" + ); + assert!(fenced(&state, "unsent") && fenced(&state, "applied")); + // What the origin answers resolves each: the old object, and the applied one. + for (bucket, key, token) in handed { + assert!(uncertain_key_is_current(&state, &bucket, &key, token)); + let size = if key == "applied" { 5 } else { 4 }; + assert!(resolve_uncertain_key( + &state, + &bucket, + &key, + token, + AuthoritativeKeyState::Present(written(size)), + )); + } + assert!(!fenced(&state, "unsent") && !fenced(&state, "applied")); + assert_eq!(indexed_size(&state, "bkt", "applied"), Some(5)); + assert_eq!(indexed_size(&state, "bkt", "unsent"), Some(4)); +} + +/// A peer cut off from the writer cannot hold the fence up: its serve-lease lapses on +/// the writer's clock, and the fence completes with the guarantee intact. +#[tokio::test] +async fn a_partitioned_peer_lapses_and_the_fence_completes() { + let net = Network::new(); + let (a_id, _a_node, a_group) = spawn_node(&net, "cut-a", "cut-b").await; + let (b_id, b_node, b_group) = spawn_node(&net, "cut-b", "cut-a").await; + let sync_a = attach(a_group, a_id, Consistency::Strong); + let _sync_b = attach(b_group, b_id.clone(), Consistency::Strong); + adopted(&sync_a, &b_id).await; + let metrics = Metrics::default(); + + // The cut: B's process runs on, but nothing reaches it or leaves it. + b_node.close().await; + let started = Instant::now(); + let pending = sync_a + .fence( + &KeyIndex::default(), + "bkt", + &["k".to_owned()], + Duration::from_millis(200), + &metrics, + ) + .await + .expect("a lapse carries the guarantee"); + assert!(pending[0].is_some()); + let waited = started.elapsed(); + assert!( + waited < TEST_LEASE * 3, + "the fence ends at the lapse, not at its own deadline ({waited:?})" + ); + let text = metrics.prometheus_text(); + assert!(text.contains("\ns3cache_write_lease_lapses 1\n")); + assert!(text.contains("\ns3cache_ack_timeouts 0\n")); +} + /// The strict-LIST barrier: after a publish, `await_fresh` on the peer /// returns true only once the event is actually applied. #[tokio::test] async fn await_fresh_reflects_the_publishers_head() { let net = Network::new(); - let (sync_a, sync_b, state, _cache) = wired_pair(&net); + let (sync_a, sync_b, state, _cache) = wired_pair(&net).await; let metrics = Metrics::default(); own_put(&sync_a, "fresh", written(7), &metrics).await; @@ -737,7 +896,7 @@ async fn await_fresh_reflects_the_publishers_head() { #[tokio::test] async fn write_tokens_upgrade_reads_to_strict() { let net = Network::new(); - let (sync_a, sync_b, state, _cache) = wired_pair(&net); + let (sync_a, sync_b, state, _cache) = wired_pair(&net).await; let metrics = Metrics::default(); let receipt = own_put(&sync_a, "tok", written(1), &metrics).await; @@ -793,7 +952,7 @@ async fn write_tokens_upgrade_reads_to_strict() { async fn in_strong_acks_an_unacked_write_is_counted_and_not_retired() { let net = Network::new(); let (_sync_a, sync_b, _state, _cache) = - wired_pair_named(&net, ("acks-a", "acks-b"), Consistency::StrongAcks); + wired_pair_named(&net, ("acks-a", "acks-b"), Consistency::StrongAcks).await; let metrics = Metrics::default(); // A was attached without an apply loop, so it publishes no ack ledger — but it // is alive, so B has to wait for it and then give up. @@ -843,7 +1002,7 @@ async fn in_strong_acks_an_unacked_write_is_counted_and_not_retired() { async fn a_leased_write_resolves_on_acks_and_the_peer_already_has_it() { let net = Network::new(); let (sync_a, _sync_b, state, _cache) = - wired_pair_named(&net, ("fast-a", "fast-b"), Consistency::Strong); + wired_pair_named(&net, ("fast-a", "fast-b"), Consistency::Strong).await; let metrics = Metrics::default(); // Nothing about a *fast* path is provable until there is somebody to be fast // against: an empty wait set resolves immediately and would pass this vacuously. @@ -883,8 +1042,8 @@ async fn a_leased_write_resolves_on_acks_and_the_peer_already_has_it() { #[tokio::test] async fn a_dropped_peers_lease_lapses_and_the_write_completes_inside_one_duration() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "lapse-a", "lapse-b"); - let (b_id, _b_node, b_group) = spawn_node(&net, "lapse-b", "lapse-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "lapse-a", "lapse-b").await; + let (b_id, _b_node, b_group) = spawn_node(&net, "lapse-b", "lapse-a").await; let sync_a = attach(a_group, a_id, Consistency::Strong); // B renews and grants but never applies: no apply loop, so no ack ledger. It is // the fail-slow reader, which is exactly the peer a lapse has to rescue. @@ -954,8 +1113,8 @@ async fn a_dropped_peers_lease_lapses_and_the_write_completes_inside_one_duratio #[tokio::test] async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { let net = Network::new(); - let (a_id, _a_node, a_group) = spawn_node(&net, "absent-a", "absent-b"); - let (b_id, b_node, b_group) = spawn_node(&net, "absent-b", "absent-a"); + let (a_id, _a_node, a_group) = spawn_node(&net, "absent-a", "absent-b").await; + let (b_id, b_node, b_group) = spawn_node(&net, "absent-b", "absent-a").await; let sync_a = attach(a_group, a_id, Consistency::Strong); let sync_b = attach(b_group, b_id.clone(), Consistency::Strong); eventually( @@ -968,12 +1127,10 @@ async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { "B is alive and granting" ); - // A crash: no departure, the node simply stops. Dropping the handles is not enough - // on its own — the receive loop keeps the node ticking until its endpoint is - // evicted, which registering the id again does. + // A crash: no departure or feed seal; close only the managed network. drop(sync_b); + b_node.close().await; drop(b_node); - drop(net.endpoint(b_id.clone())); eventually( || sync_a.absent_granters() == [b_id.clone()], "A to name B as the counted granter it is waiting on", @@ -985,7 +1142,7 @@ async fn a_granter_that_crashed_without_departing_is_named_until_it_returns() { assert!(!sync_a.may_serve_local()); // B returns under the same id (a StatefulSet pod) and is no longer named. - let (b_id, _b_node, b_group) = spawn_node(&net, "absent-b", "absent-a"); + let (b_id, _b_node, b_group) = spawn_node(&net, "absent-b", "absent-a").await; let _sync_b = attach(b_group, b_id, Consistency::Strong); eventually( || sync_a.absent_granters().is_empty(), diff --git a/src/sync/tests/fixtures.rs b/src/sync/tests/fixtures.rs new file mode 100644 index 0000000..70bce58 --- /dev/null +++ b/src/sync/tests/fixtures.rs @@ -0,0 +1,76 @@ +use std::sync::Arc; +use std::time::Duration; + +use groupnet::core::NodeId; +use groupnet::network::RouterConfig; +use groupnet::runtime::{Group, Node}; +use groupnet::transport::link::{BoundLink, LinkConfig}; +use groupnet::transport::mem::Network; + +use crate::index::KeyIndex; +use crate::metrics::Metrics; +use crate::sync::coherence::{Consistency, WriteSync}; +use crate::tier::TieredCache; + +use super::{Index, brisk, no_reconcile, test_lease}; + +pub(super) async fn spawn_node(net: &Network, id: &str, peer: &str) -> (NodeId, Node, Group) { + spawn_neighbors(net, id, &[peer]).await +} + +pub(super) async fn spawn_neighbors( + net: &Network, + id: &str, + peers: &[&str], +) -> (NodeId, Node, Group) { + let me = NodeId::new(id); + let peers = peers.iter().map(|peer| NodeId::new(*peer)).collect(); + let node = Node::builder(me.clone()) + .link(BoundLink::new( + net.endpoint(me.clone()), + LinkConfig::new(peers), + )) + .routing(RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }) + .config(brisk()) + .start() + .await + .expect("managed memory node starts"); + let group = node.join_group("s3cache"); + (me, node, group) +} + +/// A feed whose caller retains the managed node separately. +pub(super) fn attach(group: Group, me: NodeId, consistency: Consistency) -> WriteSync { + WriteSync::attach(group, me, consistency, test_lease(), None) +} + +/// A fully-wired pair: A publishes, B applies into its local state and cache. +pub(super) async fn wired_pair(net: &Network) -> (WriteSync, Arc, Index, TieredCache) { + wired_pair_named(net, ("sync-a", "sync-b"), Consistency::Strong).await +} + +/// Each returned feed owns its node, including the managed network lifetime. +pub(super) async fn wired_pair_named( + net: &Network, + ids: (&str, &str), + consistency: Consistency, +) -> (WriteSync, Arc, Index, TieredCache) { + let (a_id, a_node, a_group) = spawn_node(net, ids.0, ids.1).await; + let (b_id, b_node, b_group) = spawn_node(net, ids.1, ids.0).await; + let metrics = Arc::new(Metrics::default()); + let cache = TieredCache::new(1024 * 1024, None, metrics.clone()); + let state = Arc::new(KeyIndex::default()); + let sync_b = Arc::new(WriteSync::attach( + b_group, + b_id, + consistency, + test_lease(), + Some(b_node), + )); + sync_b.start_apply(cache.local(), state.clone(), metrics, no_reconcile()); + let sync_a = WriteSync::attach(a_group, a_id, consistency, test_lease(), Some(a_node)); + (sync_a, sync_b, state, cache) +} diff --git a/src/sync/volatile.rs b/src/sync/volatile.rs index 7b60634..caf77a1 100644 --- a/src/sync/volatile.rs +++ b/src/sync/volatile.rs @@ -1,16 +1,18 @@ //! Application facts and origin-index effects for Groupnet's volatile recovery. use std::collections::BTreeSet; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; +use groupnet::consistency::lease::CAP_LEASE; use groupnet::consistency::{ - CAP_LEASE, FrontierView, WriteToken, checked_advertised_head, + FrontierView, WriteToken, checked_advertised_head, volatile_recovery::{ - AdapterError, BoxRecoveryFuture, Mark, Peer, PeerObservation, PublicationPermit, - RecoveryAdapter, RecoveryConfig, RecoveryFallback, RecoveryHandle, RecoveryMode, - RecoveryOperation, RecoveryRearm, RecoveryStage, + AdapterError, AdapterErrorKind, Mark, Peer, PeerObservation, PublicationPermit, + RecoveryAdapter, RecoveryBuilder, RecoveryConfig, RecoveryFallback, RecoveryHandle, + RecoveryMode, RecoveryOperation, RecoveryRearm, RecoveryStage, }, }; use groupnet::core::{NodeId, Status}; @@ -125,9 +127,9 @@ impl WriteSync { .unwrap_or(2_000) .max(self.recovery_settle_floor_ms()); let config = config.unwrap_or(RecoveryConfig { - max_members: 256, - max_member_bytes: 256, - max_barrier_rounds: 4, + max_members: const { NonZeroUsize::new(256).unwrap() }, + max_member_bytes: const { NonZeroUsize::new(256).unwrap() }, + max_barrier_rounds: const { NonZeroU32::new(4).unwrap() }, // Neither bound has to fit a bucket's scan time. Every committed LIST page // reports progress, which restarts both, so a scan that keeps committing // pages runs to completion in one pass. `attempt_ms` is how long a scan may @@ -160,26 +162,25 @@ struct PreparedRecovery { } impl PreparedRecovery { - fn open_origin(&self) -> RecoveryHandle { - if let Some(policy) = self.rearm { - RecoveryHandle::open_with_rearm( - Arc::clone(&self.adapter), - self.config, - self.mode, - self.adapter.me.clone(), - self.session, - policy, - ) - } else { - RecoveryHandle::open( - Arc::clone(&self.adapter), - self.config, - self.mode, - self.adapter.me.clone(), - self.session, - ) + /// The driver's builder with this node's configuration, mode, and rearm policy. + fn builder(&self) -> RecoveryBuilder { + let builder = RecoveryHandle::builder( + Arc::clone(&self.adapter), + self.adapter.me.clone(), + self.session, + ) + .config(self.config) + .mode(self.mode); + match self.rearm { + Some(policy) => builder.rearm(policy), + None => builder, } - .expect("valid recovery configuration and Tokio runtime") + } + + fn open_origin(&self) -> RecoveryHandle { + self.builder() + .open() + .expect("valid recovery configuration and Tokio runtime") } } @@ -198,17 +199,20 @@ fn observe_group( lease: Duration, limits: RecoveryConfig, ) -> PeerObservation { - let bound = limits.max_members.checked_add(1).ok_or(AdapterError)?; + let bound = limits + .max_members + .checked_add(1) + .ok_or(AdapterErrorKind::Capacity)?; let roster = group .statuses_held_bounded(bound, limits.max_member_bytes) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; let mut peers = Vec::with_capacity(roster.len().saturating_sub(1)); for (node, status, held) in roster { if node == *me { continue; } let head = checked_advertised_head(group, &node) - .map_err(|_| AdapterError)? + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))? .map(|token| Mark { epoch: token.epoch, sequence: token.seq, @@ -245,6 +249,11 @@ impl RecoveryAdapter for CacheRecoveryAdapter { } } + fn operation_failed(&self, stage: RecoveryStage, error: AdapterError) { + let cause = std::error::Error::source(&error).map(ToString::to_string); + tracing::warn!(?stage, ?cause, "recovery operation failed: {error}"); + } + fn fell_back(&self, from: RecoveryStage, reason: RecoveryFallback) { self.metrics.recovery_fallback(); tracing::info!(?from, ?reason, "recovery fell back"); @@ -255,93 +264,92 @@ impl RecoveryAdapter for CacheRecoveryAdapter { _op: RecoveryOperation, distrust_bodies: bool, permit: PublicationPermit, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { + ) -> impl Future> + Send { + std::future::ready( permit .publish(|| { if distrust_bodies { self.local.distrust_all(); } }) - .ok_or(AdapterError) - }) + .ok_or_else(fenced), + ) } - fn rebuild_origin( + async fn rebuild_origin( &self, _op: RecoveryOperation, permit: PublicationPermit, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { - self.metrics.recovery_origin_scan(); - let buckets: BTreeSet = { - let state = self.state.read().map_err(|_| AdapterError)?; - if state.len() > MAX_RECOVERY_BUCKETS - || state.len().saturating_add(self.buckets.len()) > MAX_RECOVERY_BUCKETS - { - return Err(AdapterError); - } - let mut buckets: BTreeSet = self.buckets.iter().cloned().collect(); - buckets.extend(state.keys().cloned()); - buckets - }; - let mut generations = Vec::with_capacity(buckets.len()); - for bucket in buckets { - let generation = permit - .publish(|| begin_bucket_resync(&self.state, &bucket)) - .ok_or(AdapterError)?; - generations.push((bucket, generation)); + ) -> Result<(), AdapterError> { + self.metrics.recovery_origin_scan(); + let buckets: BTreeSet = { + let state = self + .state + .read() + .map_err(|_| AdapterError::new(AdapterErrorKind::Unavailable))?; + if state.len() > MAX_RECOVERY_BUCKETS + || state.len().saturating_add(self.buckets.len()) > MAX_RECOVERY_BUCKETS + { + return Err(AdapterErrorKind::Capacity.into()); } - for (bucket, generation) in generations { - sync_bucket_generation_guarded( - &self.client, - &self.state, - &bucket, - generation, - self.scan, - permit.clone(), - ) - .await - .map_err(|_| AdapterError)?; - } - Ok(()) - }) + let mut buckets: BTreeSet = self.buckets.iter().cloned().collect(); + buckets.extend(state.keys().cloned()); + buckets + }; + let mut generations = Vec::with_capacity(buckets.len()); + for bucket in buckets { + let generation = permit + .publish(|| begin_bucket_resync(&self.state, &bucket)) + .ok_or_else(fenced)?; + generations.push((bucket, generation)); + } + for (bucket, generation) in generations { + sync_bucket_generation_guarded( + &self.client, + &self.state, + &bucket, + generation, + self.scan, + permit.clone(), + ) + .await + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error))?; + } + Ok(()) } fn observe_peers( &self, _op: RecoveryOperation, limits: RecoveryConfig, - ) -> BoxRecoveryFuture<'_, PeerObservation> { - Box::pin(async move { - let sync = self.sync.upgrade().ok_or(AdapterError)?; - observe_group(&self.group, &sync, &self.me, self.lease, limits) + ) -> impl Future + Send { + std::future::ready(match self.sync.upgrade() { + Some(sync) => observe_group(&self.group, &sync, &self.me, self.lease, limits), + None => Err(closed()), }) } - fn wait_frontiers( + async fn wait_frontiers( &self, _op: RecoveryOperation, heads: Vec<(NodeId, Mark)>, - ) -> BoxRecoveryFuture<'_, Result<(), AdapterError>> { - Box::pin(async move { - let frontier = self.frontier().ok_or(AdapterError)?; - for (node, head) in heads { - if !frontier - .reached( - &node, - WriteToken { - epoch: head.epoch, - seq: head.sequence, - }, - ) - .await - { - return Err(AdapterError); - } + ) -> Result<(), AdapterError> { + let frontier = self.frontier().ok_or_else(closed)?; + for (node, head) in heads { + if !frontier + .reached( + &node, + WriteToken { + epoch: head.epoch, + seq: head.sequence, + }, + ) + .await + { + return Err(AdapterErrorKind::Unavailable.into()); } - Ok(()) - }) + } + Ok(()) } fn affirm(&self, _op: RecoveryOperation) -> bool { @@ -351,12 +359,24 @@ impl RecoveryAdapter for CacheRecoveryAdapter { } } +/// The operation's publication permit is no longer current. +fn fenced() -> AdapterError { + AdapterError::new(AdapterErrorKind::Fenced) +} + +/// The write feed this adapter serves has shut down or never started its apply loop. +fn closed() -> AdapterError { + AdapterError::new(AdapterErrorKind::Unavailable) +} + #[cfg(test)] mod tests { use super::*; - use groupnet::consistency::LeaseConfig; + use groupnet::consistency::lease::LeaseConfig; use groupnet::core::Config; + use groupnet::network::RouterConfig; use groupnet::runtime::Node; + use groupnet::transport::link::{BoundLink, LinkConfig}; use groupnet::transport::mem::Network; #[tokio::test] @@ -365,18 +385,33 @@ mod tests { let a = NodeId::new("recovery-a"); let b = NodeId::new("recovery-b"); let config = Config { - gossip_interval_ms: 10, - anti_entropy_interval_ms: 25, + gossip_interval_ms: 25, ..Config::default() }; - let node_a = Node::builder(a.clone(), net.endpoint(a.clone())) - .seed(b.clone()) + let routing = RouterConfig { + announce_interval: Duration::from_millis(10), + ..RouterConfig::default() + }; + let node_a = Node::builder(a.clone()) + .link(BoundLink::new( + net.endpoint(a.clone()), + LinkConfig::new(vec![b.clone()]), + )) + .routing(routing.clone()) .config(config.clone()) - .spawn(); - let node_b = Node::builder(b.clone(), net.endpoint(b.clone())) - .seed(a.clone()) + .start() + .await + .expect("A managed node starts"); + let node_b = Node::builder(b.clone()) + .link(BoundLink::new( + net.endpoint(b.clone()), + LinkConfig::new(vec![a.clone()]), + )) + .routing(routing) .config(config) - .spawn(); + .start() + .await + .expect("B managed node starts"); let group_a = node_a.join_group("s3cache"); let group_b = node_b.join_group("s3cache"); tokio::time::timeout(Duration::from_secs(3), async { @@ -394,20 +429,22 @@ mod tests { None, ); let limits = RecoveryConfig { - max_members: 4, - max_member_bytes: 64, - max_barrier_rounds: 2, + max_members: NonZeroUsize::new(4).unwrap(), + max_member_bytes: NonZeroUsize::new(64).unwrap(), + max_barrier_rounds: NonZeroU32::new(2).unwrap(), total_ms: 2_000, attempt_ms: 500, settle_ms: 100, poll_ms: 25, }; assert!(observe_group(&group_b, &sync, &b, Duration::from_millis(300), limits).is_ok()); + // Groupnet keys the default write feed's entry as `~writes/d`. + let feed_entry = "~writes/d"; group_a - .set_entry("~writes", vec![0xFF], None) + .set_entry(feed_entry, vec![0xFF], None) .expect("malformed entry enqueued"); tokio::time::timeout(Duration::from_secs(3), async { - while group_b.node_entry(&a, "~writes").is_none() { + while group_b.node_entry(&a, feed_entry).is_none() { tokio::task::yield_now().await; } }) diff --git a/src/sync/volatile/fleet.rs b/src/sync/volatile/fleet.rs index 111d1eb..f18028a 100644 --- a/src/sync/volatile/fleet.rs +++ b/src/sync/volatile/fleet.rs @@ -1,5 +1,6 @@ //! S3 index facts supplied to Groupnet's one optional bootstrap worker. +use std::num::NonZeroUsize; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -16,7 +17,9 @@ use groupnet::core::volatile_bootstrap::{ use crate::index::fleet::FleetDonorImage; -use super::{AdapterError, CacheRecoveryAdapter, RecoveryAdapter}; +use super::{ + AdapterError, AdapterErrorKind, CacheRecoveryAdapter, RecoveryAdapter, closed, fenced, +}; mod capture; mod donor; @@ -25,9 +28,9 @@ mod runtime; pub(crate) use runtime::FleetListenerGuard; pub(super) use runtime::open_recovery; -const MAX_ROSTER: usize = 64; -const MAX_ROSTER_ID_BYTES: usize = 256; -const MAX_ROSTER_METADATA: usize = 64 << 10; +const MAX_ROSTER: NonZeroUsize = NonZeroUsize::new(64).unwrap(); +const MAX_ROSTER_ID_BYTES: NonZeroUsize = NonZeroUsize::new(256).unwrap(); +const MAX_ROSTER_METADATA: NonZeroUsize = NonZeroUsize::new(64 << 10).unwrap(); /// Consumer index/source evidence. Claim election, bulk retry, and lifetime /// decisions remain in the existing Groupnet recovery worker. @@ -87,7 +90,7 @@ impl FleetStatePort { || universe.len() > capture::IMAGE_CAPS.buckets || universe.windows(2).any(|pair| pair[0] >= pair[1]) { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } Ok(Self { adapter, @@ -138,7 +141,7 @@ impl FleetStatePort { // cause a second origin LIST. Once local recovery reaches Ready, the // same Groupnet worker attempts one finite guarded index recapture. if !request.permit.valid() { - return Err(AdapterError); + return Err(fenced()); } Ok(LocalCaptureOutcome::LocalOnly) } @@ -174,7 +177,7 @@ impl FleetStatePort { request: ReadyCaptureRequest, admission: &ByteAdmission, ) -> Result, AdapterError> { - let sync = self.adapter.sync.upgrade().ok_or(AdapterError)?; + let sync = self.adapter.sync.upgrade().ok_or_else(closed)?; if !self.listener_alive() || !self .source_matches( @@ -186,7 +189,7 @@ impl FleetStatePort { .await { tracing::info!("fleet Ready recapture declined: source roster changed"); - return Err(AdapterError); + return Err(fenced()); } let started = Instant::now(); let id = capture::next_id( @@ -214,7 +217,7 @@ impl FleetStatePort { || !sync.mode_allows_local() || !self.listener_alive() { - return Err(AdapterError); + return Err(fenced()); } prepared.attach( &self.adapter.state, @@ -225,7 +228,7 @@ impl FleetStatePort { request.wake, ) }) - .ok_or(AdapterError) + .ok_or_else(fenced) .flatten() .inspect_err(|_| { tracing::info!( @@ -241,7 +244,7 @@ impl FleetStatePort { .await { tracing::info!("fleet Ready recapture declined: roster changed while encoding"); - return Err(AdapterError); + return Err(fenced()); } let captured = request .guard @@ -250,11 +253,11 @@ impl FleetStatePort { || !sync.mode_allows_local() || !self.listener_alive() { - return Err(AdapterError); + return Err(fenced()); } capture::finish(pending) }) - .ok_or(AdapterError) + .ok_or_else(fenced) .flatten(); if captured.is_ok() { tracing::info!( diff --git a/src/sync/volatile/fleet/capture.rs b/src/sync/volatile/fleet/capture.rs index 2891f0c..c234079 100644 --- a/src/sync/volatile/fleet/capture.rs +++ b/src/sync/volatile/fleet/capture.rs @@ -1,5 +1,6 @@ //! Finite donor C capture, with all large encoding outside the index lock. +use std::num::NonZeroUsize; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -16,7 +17,7 @@ use tokio::sync::Notify; use crate::index::KeyIndex; use crate::index::fleet::{FleetDonorImage, ImageCaps, PendingFleetCapture}; -use super::super::AdapterError; +use super::super::{AdapterError, AdapterErrorKind, fenced}; static NEXT_CAPTURE: AtomicU64 = AtomicU64::new(1); @@ -35,23 +36,28 @@ pub(super) const IMAGE_CAPS: ImageCaps = ImageCaps { name_bytes: 1_024, }; +/// `n` as a bound Groupnet accepts; a zero bound fails to compile. +pub(super) const fn bound(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).expect("journal bounds are nonzero") +} + pub(super) const JOURNAL_CONFIG: JournalConfig = JournalConfig { - max_encoded_bytes: IMAGE_CAPS.bytes, - max_decoded_bytes: IMAGE_CAPS.decoded_bytes, - max_events: 1_024, - max_suffix_bytes: 8 << 20, - max_event_bytes: 4_096, - max_identity_bytes: 256, - max_followers: 2, - max_follower_id_bytes: 256, - max_cuts: 64, - max_cut_bytes: 4_096, - max_members: 64, - max_membership_bytes: 16 << 10, - max_scope_bytes: 8_192, - max_batch_events: 32, - max_batch_bytes: 128 << 10, - max_inflight_bytes: 256 << 10, + max_encoded_bytes: bound(IMAGE_CAPS.bytes), + max_decoded_bytes: bound(IMAGE_CAPS.decoded_bytes), + max_events: bound(1_024), + max_suffix_bytes: bound(8 << 20), + max_event_bytes: bound(4_096), + max_identity_bytes: bound(256), + max_followers: bound(2), + max_follower_id_bytes: bound(256), + max_cuts: bound(64), + max_cut_bytes: bound(4_096), + max_members: bound(64), + max_membership_bytes: bound(16 << 10), + max_scope_bytes: bound(8_192), + max_batch_events: bound(32), + max_batch_bytes: bound(128 << 10), + max_inflight_bytes: bound(256 << 10), max_total_ms: 600_000, // A 108 MB production image is about 1,700 sequential 64 KiB chunks; give a // follower on a loaded node minutes, not one, before its reservation expires. @@ -67,9 +73,9 @@ pub(super) fn next_id( .try_update(Ordering::AcqRel, Ordering::Acquire, |old| { old.checked_add(1) }) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; if serial == 0 || generation == 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } Ok(CaptureId { scope, @@ -91,11 +97,13 @@ pub(super) fn prepare( budget: &ByteAdmission, id: CaptureId, ) -> Result { - let suffix_bytes = DonorJournal::storage_bound(JOURNAL_CONFIG).map_err(|_| AdapterError)?; + let suffix_bytes = DonorJournal::storage_bound(JOURNAL_CONFIG) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::InvalidInput, error))?; let suffix = budget .reserve(AdmissionClass::Suffix, suffix_bytes) - .map_err(|_| AdapterError)?; - let journal = DonorJournal::new(JOURNAL_CONFIG, id).map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; + let journal = DonorJournal::new(JOURNAL_CONFIG, id) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::InvalidInput, error))?; Ok(PreparedCapture { journal, suffix }) } @@ -122,7 +130,7 @@ impl PreparedCapture { IMAGE_CAPS, clock, ) - .map_err(|_| AdapterError) + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity)) } } @@ -136,17 +144,17 @@ pub(super) async fn encode( tokio::task::spawn_blocking(move || { pending .encode(&universe, IMAGE_CAPS) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; Ok(pending) }) .await - .map_err(|_| AdapterError)? + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error))? } pub(super) fn finish( pending: PendingFleetCapture, ) -> Result, AdapterError> { - pending.finish().map_err(|_| AdapterError) + pending.finish().map_err(|_| fenced()) } pub(super) fn same_roster( diff --git a/src/sync/volatile/fleet/donor.rs b/src/sync/volatile/fleet/donor.rs index 375cb1b..cdbcfc6 100644 --- a/src/sync/volatile/fleet/donor.rs +++ b/src/sync/volatile/fleet/donor.rs @@ -1,27 +1,36 @@ //! Groupnet donor callbacks: complete guarded C and immutable bounded chunks. use std::mem::size_of; -use std::time::Instant; +use std::num::NonZeroUsize; use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionClass, Admitted, ByteAdmission, }; use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::BootstrapStatePort; use groupnet::consistency::volatile_recovery::bootstrap::ports::{ - DonorCapture, LocalCaptureOutcome, LocalCaptureRequest, ReadyCaptureRequest, TransferContext, + DonorCapture, EffectContext, LocalCaptureOutcome, LocalCaptureRequest, ReadyCaptureRequest, TransferResources, }; -use groupnet::consistency::volatile_recovery::{ - AdapterError, BoxRecoveryFuture, PublicationPermit, -}; +use groupnet::consistency::volatile_recovery::{AdapterError, AdapterErrorKind}; use groupnet::core::volatile_bootstrap::journal::{AttachToken, JournalBatch, JournalCursor}; use groupnet::core::volatile_bootstrap::transfer::{TransferEffect, TransferEvent, TransferOffer}; use crate::index::fleet::{FleetDonorImage, FleetStage}; +use crate::sync::volatile::fenced; use super::{FleetStatePort, capture}; -const CHUNK_BYTES: usize = 64 << 10; +const CHUNK_BYTES: NonZeroUsize = NonZeroUsize::new(64 << 10).unwrap(); + +/// A request or argument outside this donor's bounds. +fn invalid() -> AdapterError { + AdapterError::new(AdapterErrorKind::InvalidInput) +} + +/// An arithmetic bound or memory admission refused the work. +fn capacity() -> AdapterError { + AdapterError::new(AdapterErrorKind::Capacity) +} impl BootstrapStatePort for FleetStatePort { type Image = FleetDonorImage; @@ -31,20 +40,20 @@ impl BootstrapStatePort for FleetStatePort { // refuses any concurrent final effect at the guarded install. type NativeBuffer = (); - fn build_local_capture<'a>( - &'a self, + async fn build_local_capture( + &self, request: LocalCaptureRequest, - admission: &'a ByteAdmission, - ) -> BoxRecoveryFuture<'a, Result, AdapterError>> { - Box::pin(self.initial_capture(request, admission)) + admission: &ByteAdmission, + ) -> Result, AdapterError> { + self.initial_capture(request, admission).await } - fn recapture_current_index<'a>( - &'a self, + async fn recapture_current_index( + &self, request: ReadyCaptureRequest, - admission: &'a ByteAdmission, - ) -> BoxRecoveryFuture<'a, Result, AdapterError>> { - Box::pin(self.ready_capture(request, admission)) + admission: &ByteAdmission, + ) -> Result, AdapterError> { + self.ready_capture(request, admission).await } fn retire_local_capture(&self, capture: &DonorCapture) { @@ -54,15 +63,15 @@ impl BootstrapStatePort for FleetStatePort { fn image_offer( &self, capture: &DonorCapture, - max_metadata_bytes: usize, + max_metadata_bytes: NonZeroUsize, admission: &ByteAdmission, ) -> Result, AdapterError> { if !capture.is_active() || !capture::still_attached(&self.adapter.state, capture) { - return Err(AdapterError); + return Err(fenced()); } - let bytes = capture.image().as_bytes().len(); - if bytes == 0 || bytes > capture::IMAGE_CAPS.bytes || max_metadata_bytes == 0 { - return Err(AdapterError); + let bytes = NonZeroUsize::new(capture.image().as_bytes().len()).ok_or_else(invalid)?; + if bytes.get() > capture::IMAGE_CAPS.bytes { + return Err(capacity()); } let chunks = bytes.div_ceil(CHUNK_BYTES); // Charge the complete retained DTO, both scope copies, cut vectors, @@ -70,27 +79,27 @@ impl BootstrapStatePort for FleetStatePort { // entering the journal and allocating any response-owned metadata. let headers = size_of::() .checked_add( - capture::JOURNAL_CONFIG.max_members + capture::JOURNAL_CONFIG.max_members.get() * size_of::(), ) .and_then(|n| { n.checked_add( - capture::JOURNAL_CONFIG.max_cuts + capture::JOURNAL_CONFIG.max_cuts.get() * size_of::(), ) }) - .ok_or(AdapterError)?; + .ok_or_else(capacity)?; let charge = max_metadata_bytes - .checked_mul(4) + .checked_mul(NonZeroUsize::new(4).ok_or_else(capacity)?) .and_then(|n| n.checked_add(headers)) - .ok_or(AdapterError)?; + .ok_or_else(capacity)?; let permit = admission .reserve(AdmissionClass::Inflight, charge) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; // The decoded charge finished at C is the measured image charge the // follower must reserve and decode under. let offer = capture.ingress().with_journal(|journal| { - let decoded_bytes = journal.image_charge()?.decoded_bytes; + let decoded_bytes = journal.image_charge()?.image.decoded_bytes; let id = journal.id().clone(); let members = journal.image_members().to_vec(); let cuts = journal.image_cuts().to_vec(); @@ -109,7 +118,7 @@ impl BootstrapStatePort for FleetStatePort { cuts, }) }); - let offer = offer.ok_or(AdapterError)?; + let offer = offer.ok_or_else(fenced)?; let variable = offer .capture .scope @@ -128,9 +137,12 @@ impl BootstrapStatePort for FleetStatePort { .iter() .try_fold(n, |sum, cut| sum.checked_add(cut.writer.len())) }) - .ok_or(AdapterError)?; - if variable > max_metadata_bytes || !capture.is_active() { - return Err(AdapterError); + .ok_or_else(capacity)?; + if variable > max_metadata_bytes.get() { + return Err(capacity()); + } + if !capture.is_active() { + return Err(fenced()); } Ok(permit.hold(offer)) } @@ -139,97 +151,102 @@ impl BootstrapStatePort for FleetStatePort { &self, capture: &DonorCapture, sequence: usize, - max_bytes: usize, + max_bytes: NonZeroUsize, admission: &ByteAdmission, ) -> Result>, AdapterError> { - if !capture.is_active() - || !capture::still_attached(&self.adapter.state, capture) - || max_bytes == 0 - || max_bytes > CHUNK_BYTES - { - return Err(AdapterError); + if !capture.is_active() || !capture::still_attached(&self.adapter.state, capture) { + return Err(fenced()); } - let start = sequence.checked_mul(CHUNK_BYTES).ok_or(AdapterError)?; + if max_bytes > CHUNK_BYTES { + return Err(invalid()); + } + let start = sequence + .checked_mul(CHUNK_BYTES.get()) + .ok_or_else(invalid)?; let end = start - .checked_add(CHUNK_BYTES) - .ok_or(AdapterError)? + .checked_add(CHUNK_BYTES.get()) + .ok_or_else(invalid)? .min(capture.image().as_bytes().len()); let slice = capture .image() .as_bytes() .get(start..end) - .ok_or(AdapterError)?; - if slice.is_empty() || slice.len() > max_bytes { - return Err(AdapterError); + .ok_or_else(invalid)?; + if slice.is_empty() || slice.len() > max_bytes.get() { + return Err(invalid()); } let permit = admission .reserve(AdmissionClass::Encoded, max_bytes) - .map_err(|_| AdapterError)?; + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; let mut copy = Vec::new(); copy.try_reserve_exact(slice.len()) - .map_err(|_| AdapterError)?; - if copy.capacity() > max_bytes { - return Err(AdapterError); + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error))?; + if copy.capacity() > max_bytes.get() { + return Err(capacity()); } copy.extend_from_slice(slice); Ok(permit.hold(copy)) } - fn execute_local<'a>( - &'a self, - context: &'a TransferContext, + async fn execute_local( + &self, effect: TransferEffect, - resources: &'a mut TransferResources, - admission: &'a ByteAdmission, - permit: Option, - deadline: Instant, - ) -> BoxRecoveryFuture<'a, Result>, AdapterError>> { - Box::pin(self.execute_follower(context, effect, resources, admission, permit, deadline)) + resources: &mut TransferResources, + context: EffectContext<'_>, + ) -> Result>, AdapterError> { + self.execute_follower(effect, resources, context).await } - fn store_chunk<'a>( - &'a self, + fn store_chunk( + &self, sequence: usize, - chunk: &'a Admitted>, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result> { - Box::pin(async move { - let stage = resources.stage.as_mut().ok_or(AdapterError)?; - stage - .stage_mut() - .store_chunk(sequence, chunk.get()) - .map_err(|_| AdapterError)?; - Ok(stage.reserved_bytes().1) - }) + chunk: &Admitted>, + resources: &mut TransferResources, + ) -> impl Future> + Send { + std::future::ready( + resources + .stage + .as_mut() + .ok_or_else(fenced) + .and_then(|stage| { + stage + .stage_mut() + .store_chunk(sequence, chunk.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; + Ok(stage.reserved_bytes().1) + }), + ) } - fn stage_batch<'a>( - &'a self, - batch: &'a Admitted, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result<(), AdapterError>> { - Box::pin(async move { + fn stage_batch( + &self, + batch: &Admitted, + resources: &mut TransferResources, + ) -> impl Future> + Send { + std::future::ready( resources .stage .as_mut() - .ok_or(AdapterError)? - .stage_mut() - .stage_batch(batch.get()) - .map_err(|_| AdapterError) - }) + .ok_or_else(fenced) + .and_then(|stage| { + stage + .stage_mut() + .stage_batch(batch.get()) + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed)) + }), + ) } - fn detach<'a>( - &'a self, + fn detach( + &self, token: AttachToken, - resources: &'a mut TransferResources, - ) -> BoxRecoveryFuture<'a, Result<(), AdapterError>> { - Box::pin(async move { - if resources.attachment.as_ref() != Some(&token) { - return Err(AdapterError); - } + resources: &mut TransferResources, + ) -> impl Future> + Send { + std::future::ready(if resources.attachment.as_ref() == Some(&token) { resources.attachment.take(); Ok(()) + } else { + Err(fenced()) }) } } diff --git a/src/sync/volatile/fleet/follower.rs b/src/sync/volatile/fleet/follower.rs index b9b7dec..e474943 100644 --- a/src/sync/volatile/fleet/follower.rs +++ b/src/sync/volatile/fleet/follower.rs @@ -1,6 +1,7 @@ //! Private follower staging and one guarded native-feed/index handoff. use std::mem::size_of; +use std::num::NonZeroUsize; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; @@ -8,15 +9,16 @@ use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionClass, Admitted, ByteAdmission, Reservation, }; use groupnet::consistency::volatile_recovery::bootstrap::ports::{ - StageResources, TransferContext, TransferResources, + EffectContext, StageResources, TransferResources, }; -use groupnet::consistency::volatile_recovery::{AdapterError, PublicationPermit}; +use groupnet::consistency::volatile_recovery::{AdapterError, AdapterErrorKind}; use groupnet::core::volatile_bootstrap::journal::AttachToken; use groupnet::core::volatile_bootstrap::transfer::{ NativeCoverageReceipt, NativeHandoffReceipt, TransferEffect, TransferEvent, }; use crate::index::fleet::{FleetStage, IMAGE_SCHEMA, InstallRefusal}; +use crate::sync::volatile::fenced; use super::{FleetStatePort, capture}; @@ -31,16 +33,16 @@ impl FleetStatePort { // B, proven cuts, membership, and handoff metadata coexist with // the original effect and retained stage. Charge this physical // result copy separately from Groupnet's core event envelope. - (64_usize << 10) - .checked_mul(4) + NonZeroUsize::new(64_usize << 10) + .and_then(|n| n.checked_mul(NonZeroUsize::new(4)?)) .and_then(|n| n.checked_add(size_of::())) - .ok_or(AdapterError)? + .ok_or(AdapterErrorKind::Capacity)? } else { - size_of::().max(1) + NonZeroUsize::new(size_of::()).unwrap_or(NonZeroUsize::MIN) }; admission .reserve(AdmissionClass::Inflight, bytes) - .map_err(|_| AdapterError) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Capacity, error)) } #[expect( @@ -49,15 +51,19 @@ impl FleetStatePort { )] pub(super) async fn execute_follower( &self, - context: &TransferContext, effect: TransferEffect, resources: &mut TransferResources, - admission: &ByteAdmission, - permit: Option, - deadline: Instant, + context: EffectContext<'_>, ) -> Result>, AdapterError> { + let EffectContext { + transfer: context, + admission, + permit, + deadline, + .. + } = context; if Instant::now() >= deadline { - return Err(AdapterError); + return Err(AdapterErrorKind::Expired.into()); } match effect { TransferEffect::ReserveStage { @@ -68,15 +74,15 @@ impl FleetStatePort { decoded_bytes, } => { if resources.stage.is_some() - || encoded_bytes > capture::IMAGE_CAPS.bytes - || decoded_bytes > capture::IMAGE_CAPS.decoded_bytes + || encoded_bytes.get() > capture::IMAGE_CAPS.bytes + || decoded_bytes.get() > capture::IMAGE_CAPS.decoded_bytes { tracing::debug!( encoded_bytes, decoded_bytes, "fleet peer image declined: over the image codec ceilings" ); - return Err(AdapterError); + return Err(AdapterErrorKind::Capacity.into()); } let reserved = admission .reserve(AdmissionClass::Encoded, encoded_bytes) @@ -85,30 +91,30 @@ impl FleetStatePort { .reserve(AdmissionClass::Decoded, decoded_bytes) .map(|decoded| (encoded, decoded)) }); - let Ok((encoded, decoded)) = reserved else { + let (encoded, decoded) = reserved.map_err(|error| { tracing::debug!( encoded_bytes, decoded_bytes, "fleet peer image declined: memory admission" ); - return Err(AdapterError); - }; + AdapterError::with_source(AdapterErrorKind::Capacity, error) + })?; let stage = FleetStage::new( - encoded_bytes, - chunks, + encoded_bytes.get(), + chunks.get(), image_cut, - decoded_bytes, + decoded_bytes.get(), capture::IMAGE_CAPS, - capture::JOURNAL_CONFIG.max_event_bytes, + capture::JOURNAL_CONFIG.max_event_bytes.get(), ) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::InvalidInput))?; resources.stage = Some(StageResources::new(stage, encoded, decoded)?); Ok(Some( Self::event_charge(admission, false)?.hold(TransferEvent::StageReserved { op }), )) } TransferEffect::VerifyImage { op, commitment } => { - let mut stage = resources.stage.take().ok_or(AdapterError)?; + let mut stage = resources.stage.take().ok_or_else(fenced)?; // Hash and full map decode are bounded but potentially large. // The blocking task owns the stage and both permits until it // exits, even if the recovery worker is cancelled. @@ -116,11 +122,11 @@ impl FleetStatePort { stage .stage_mut() .verify(commitment) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Malformed))?; Ok::<_, AdapterError>(stage) }) .await - .map_err(|_| AdapterError) + .map_err(|error| AdapterError::with_source(AdapterErrorKind::Unavailable, error)) .flatten() .inspect_err(|_| { tracing::debug!("fleet peer image declined: commitment or decode failed"); @@ -142,9 +148,9 @@ impl FleetStatePort { .source_matches(op, &receipt.members, admission, deadline) .await { - return Err(AdapterError); + return Err(fenced()); } - let stage = resources.stage.as_mut().ok_or(AdapterError)?; + let stage = resources.stage.as_mut().ok_or_else(fenced)?; match stage.stage_mut().check_coverage( &self.adapter.state, &receipt.covered_cuts, @@ -165,7 +171,7 @@ impl FleetStatePort { refusal = %why, "fleet peer image declined: live index incompatible" ); - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } } let charge = Self::event_charge(admission, true)?; @@ -182,15 +188,15 @@ impl FleetStatePort { )) } TransferEffect::InstallCandidate { op, coverage } => { - let permit = permit.ok_or(AdapterError)?.restricted_to(deadline); - let attachment = resources.attachment.as_ref().ok_or(AdapterError)?; + let permit = permit.ok_or_else(fenced)?.restricted_to(deadline); + let attachment = resources.attachment.as_ref().ok_or_else(fenced)?; if coverage.parent != context.parent || coverage.barrier.reservation != attachment.reservation || coverage.barrier.attach_operation != attachment.operation || coverage.staged_through != coverage.barrier.cursor || coverage.buffered_bytes != 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } // Membership can change after the B coverage observation even // when the local index publication serial stays unchanged. @@ -199,7 +205,7 @@ impl FleetStatePort { .source_matches(op, &coverage.members, admission, deadline) .await { - return Err(AdapterError); + return Err(fenced()); } let charge = Self::event_charge(admission, true)?; let continued_cuts = coverage.proven_cuts.clone(); @@ -208,11 +214,11 @@ impl FleetStatePort { .try_update(Ordering::AcqRel, Ordering::Acquire, |old| { old.checked_add(1) }) - .map_err(|_| AdapterError)?; + .map_err(|_| AdapterError::new(AdapterErrorKind::Capacity))?; if applier_generation == 0 { - return Err(AdapterError); + return Err(AdapterErrorKind::Capacity.into()); } - let stage = resources.stage.as_mut().ok_or(AdapterError)?; + let stage = resources.stage.as_mut().ok_or_else(fenced)?; let installed = permit .publish(|| { stage.stage_mut().install_into( @@ -221,7 +227,7 @@ impl FleetStatePort { &self.universe, ) }) - .ok_or(AdapterError)?; + .ok_or_else(fenced)?; match installed { // The image moved into the live index; release the // emptied stage and its private memory permits. @@ -251,7 +257,7 @@ impl FleetStatePort { refusal = %why, "fleet peer image declined at install: live index incompatible" ); - return Err(AdapterError); + return Err(AdapterErrorKind::InvalidInput.into()); } } let handoff = NativeHandoffReceipt { @@ -274,7 +280,7 @@ impl FleetStatePort { resources.native_overlap.take(); Ok(None) } - _ => Err(AdapterError), + _ => Err(AdapterErrorKind::Unsupported.into()), } } } diff --git a/src/sync/volatile/fleet/runtime.rs b/src/sync/volatile/fleet/runtime.rs index 39e57ae..aae34f3 100644 --- a/src/sync/volatile/fleet/runtime.rs +++ b/src/sync/volatile/fleet/runtime.rs @@ -1,17 +1,24 @@ //! One TCP listener and claim/transfer child of the recovery worker. +use std::num::NonZeroU32; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use groupnet::consistency::QueueCapacity; use groupnet::consistency::volatile_recovery::RecoveryHandle; use groupnet::consistency::volatile_recovery::bootstrap::admission::{ AdmissionLimits, ByteAdmission, }; -use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::BulkDonorPort; +use groupnet::consistency::volatile_recovery::bootstrap::bulk_adapter::{ + BulkDonorOptions, BulkDonorPort, +}; use groupnet::consistency::volatile_recovery::bootstrap::bulk_wire::{ BootstrapBulkListener, BulkLimits, PhaseLimits, WireLimits, }; -use groupnet::consistency::volatile_recovery::bootstrap::native_claims::NativeClaimSource; +use groupnet::consistency::volatile_recovery::bootstrap::native_claims::{ + NativeClaimOptions, NativeClaimSource, +}; use groupnet::consistency::volatile_recovery::bootstrap::ports::BootstrapCapabilities; use groupnet::consistency::volatile_recovery::bootstrap::report::{ BootstrapDecision, BootstrapObserver, @@ -21,7 +28,7 @@ use groupnet::consistency::volatile_recovery::bootstrap::session::{ }; use groupnet::core::NodeId; use groupnet::core::volatile_bootstrap::transfer::TransferConfig; -use groupnet::core::volatile_bootstrap::{BootstrapConfig, BootstrapScope}; +use groupnet::core::volatile_bootstrap::{BootstrapConfig, BootstrapScope, PresenceIdentity}; use groupnet::transport::bulk::DataPlane; use groupnet::transport::tcp::TcpBulkTransport; use tokio::sync::watch; @@ -31,6 +38,7 @@ use crate::metrics::Metrics; use crate::sync::coherence::WriteSync; use crate::sync::fleet::fresh_boot; +use super::capture::bound; use super::{FleetStatePort, capture}; use crate::sync::volatile::{CacheRecoveryAdapter, PreparedRecovery}; @@ -97,7 +105,7 @@ fn claim_config(recovery: &PreparedRecovery) -> Option { let total_ms = recovery.config.total_ms.min(60_000); (total_ms >= 5_000).then_some(BootstrapConfig { max_members: capture::JOURNAL_CONFIG.max_members, - max_member_bytes: 256, + max_member_bytes: bound(256), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, // Give a newly observed Ready donor's native TTL claim time to // propagate during the existing core settle phase. This remains @@ -111,36 +119,36 @@ fn claim_config(recovery: &PreparedRecovery) -> Option { }) } -fn transfer_config() -> TransferConfig { - TransferConfig { - expected_schema: crate::index::fleet::IMAGE_SCHEMA, - max_metadata_bytes: 64 << 10, - max_encoded_bytes: capture::IMAGE_CAPS.bytes, - max_decoded_bytes: capture::IMAGE_CAPS.decoded_bytes, - max_chunk_bytes: 64 << 10, - max_chunks: capture::IMAGE_CAPS.bytes / (64 << 10), +fn transfer_config() -> Option { + Some(TransferConfig { + expected_schema: NonZeroU32::new(crate::index::fleet::IMAGE_SCHEMA)?, + max_metadata_bytes: bound(64 << 10), + max_encoded_bytes: bound(capture::IMAGE_CAPS.bytes), + max_decoded_bytes: bound(capture::IMAGE_CAPS.decoded_bytes), + max_chunk_bytes: bound(64 << 10), + max_chunks: bound(capture::IMAGE_CAPS.bytes / (64 << 10)), max_batch_bytes: capture::JOURNAL_CONFIG.max_batch_bytes, max_batch_events: capture::JOURNAL_CONFIG.max_batch_events, max_replay_events: capture::JOURNAL_CONFIG.max_events, - max_native_buffer_bytes: 1 << 20, + max_native_buffer_bytes: bound(1 << 20), max_members: capture::JOURNAL_CONFIG.max_members, max_cuts: capture::JOURNAL_CONFIG.max_cuts, coverage_poll_ms: 100, - } + }) } fn bulk_limits() -> BulkLimits { BulkLimits { wire: WireLimits { - max_frame_bytes: 256 << 10, + max_frame_bytes: bound(256 << 10), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, - max_node_bytes: 256, - max_payload_bytes: 192 << 10, + max_node_bytes: bound(256), + max_payload_bytes: bound(192 << 10), }, phase: PhaseLimits { - max_body_bytes: 128 << 10, + max_body_bytes: bound(128 << 10), max_scope_bytes: capture::JOURNAL_CONFIG.max_scope_bytes, - max_node_bytes: 256, + max_node_bytes: bound(256), max_writer_bytes: capture::JOURNAL_CONFIG.max_cut_bytes, max_cuts: capture::JOURNAL_CONFIG.max_cuts, max_members: capture::JOURNAL_CONFIG.max_members, @@ -148,7 +156,7 @@ fn bulk_limits() -> BulkLimits { max_identity_bytes: capture::JOURNAL_CONFIG.max_identity_bytes, max_effect_bytes: capture::JOURNAL_CONFIG.max_event_bytes, }, - server_request_ms: 3_000, + server_request: Duration::from_millis(3_000), } } @@ -157,13 +165,13 @@ fn bulk_limits() -> BulkLimits { /// safety cap. One node is a donor or a follower within a recovery, never both. fn admission() -> Option { ByteAdmission::new(AdmissionLimits { - max_total_bytes: (256 << 20) + (1 << 30) + (80 << 20), + max_total_bytes: bound((256 << 20) + (1 << 30) + (80 << 20)), max_encoded_bytes: capture::IMAGE_CAPS.bytes, max_decoded_bytes: capture::IMAGE_CAPS.decoded_bytes, max_suffix_bytes: 32 << 20, max_native_overlap_bytes: 16 << 20, max_inflight_bytes: 32 << 20, - max_reservations: 512, + max_reservations: bound(512), }) .ok() } @@ -192,15 +200,17 @@ pub(in crate::sync::volatile) async fn open_recovery( let fleet = prepared.fleet.as_ref()?; let scope: BootstrapScope = required("scope", fleet.scope(&prepared.adapter.buckets).ok())?; let claim = required("claim configuration", claim_config(prepared))?; - let claim = required("claim policy", claim.validate().ok())?; - let transfer = required("transfer policy", transfer_config().validate().ok())?; - let limits = required("bulk limits", bulk_limits().validate().ok())?; + required("claim policy", claim.validate().ok())?; + let transfer = required("transfer schema", transfer_config())?; + required("transfer policy", transfer.validate().ok())?; + let limits = bulk_limits(); + required("bulk limits", limits.validate().ok())?; let budget = required("admission", admission())?; let suffix_bytes = groupnet::core::volatile_bootstrap::journal::DonorJournal::storage_bound( capture::JOURNAL_CONFIG, ) .ok()?; - if suffix_bytes > (32 << 20) { + if suffix_bytes.get() > (32 << 20) { tracing::warn!("fleet suffix exceeds shared admission cap; using guarded origin recovery"); return None; } @@ -225,11 +235,13 @@ pub(in crate::sync::volatile) async fn open_recovery( "native claim source", NativeClaimSource::new( sync.group.clone(), - scope.clone(), - claim, - 32 << 10, - 64 << 10, budget.clone(), + NativeClaimOptions { + scope: scope.clone(), + policy: claim, + max_key_bytes: bound(32 << 10), + max_value_bytes: bound(64 << 10), + }, ) .ok(), )?); @@ -251,9 +263,11 @@ pub(in crate::sync::volatile) async fn open_recovery( Arc::clone(&state), plane.clone(), budget.clone(), - scope.clone(), - transfer, - limits, + BulkDonorOptions { + scope: scope.clone(), + policy: transfer, + limits, + }, ) .ok(), )?); @@ -265,8 +279,8 @@ pub(in crate::sync::volatile) async fn open_recovery( let runtime_config = BootstrapRuntimeConfig { claim, transfer, - max_claim_metadata_bytes: 64 << 10, - donor_inbox_capacity: 32, + max_claim_metadata_bytes: bound(64 << 10), + donor_inbox_capacity: required("donor inbox", QueueCapacity::new(32))?, require_participation: true, }; let (child, sender) = required( @@ -275,9 +289,11 @@ pub(in crate::sync::volatile) async fn open_recovery( capabilities, runtime_config, scope, - sync.me.clone(), - boot, - prepared.session, + PresenceIdentity { + node: sync.me.clone(), + boot, + session: prepared.session, + }, ) .ok(), )?; @@ -287,27 +303,7 @@ pub(in crate::sync::volatile) async fn open_recovery( )?; let bootstrap = Box::new(child.with_observer(Arc::new(DecisionLog(Arc::clone(&prepared.adapter.metrics))))); - let handle = if let Some(rearm) = prepared.rearm { - RecoveryHandle::open_with_bootstrap_and_rearm( - Arc::clone(&prepared.adapter), - prepared.config, - prepared.mode, - sync.me.clone(), - prepared.session, - rearm, - bootstrap, - ) - } else { - RecoveryHandle::open_with_bootstrap( - Arc::clone(&prepared.adapter), - prepared.config, - prepared.mode, - sync.me.clone(), - prepared.session, - bootstrap, - ) - } - .ok()?; + let handle = prepared.builder().bootstrap(bootstrap).open().ok()?; let (shutdown, receiver) = watch::channel(false); let task = tokio::spawn(async move { if let Err(error) = listener.run(receiver).await { diff --git a/src/sync/wire.rs b/src/sync/wire.rs index ad8ec66..f3b4c59 100644 --- a/src/sync/wire.rs +++ b/src/sync/wire.rs @@ -15,11 +15,12 @@ pub(super) fn parse_token(value: &str) -> Option<(&str, u64, u64)> { /// Prefix identifying the current event envelope: a postcard `IndexEvent`. Bytes /// without it are not this protocol and are rejected rather than interpreted as -/// another shape. Peers upgrade together, so the retired bincode envelope (`0xFF`) -/// is not read: mixed-version fleets are unsupported, as for every peer protocol. -pub(super) const WIRE_MAGIC: u8 = 0xFE; +/// another shape. Peers upgrade together, so the retired envelopes — bincode (`0xFF`) +/// and the postcard event before the pre-write fence (`0xFE`) — are not read: +/// mixed-version fleets are unsupported, as for every peer protocol. +pub(super) const WIRE_MAGIC: u8 = 0xFD; -/// What one durable write did, as advertised to peers. +/// What one write did, or is about to do, as advertised to peers. #[derive(Serialize, Deserialize)] pub(crate) enum IndexOp { /// The key now holds an object. @@ -41,6 +42,13 @@ pub(crate) enum IndexOp { }, /// The key was deleted. Del, + /// A `strong` write of the key is about to reach the origin. Until the writer + /// settles it, a reader answers the key from the origin: a writer that crashes + /// after the origin applied the write cannot tell anyone what it did. + Pending, + /// The origin refused the pending write, or it was never sent: the key is as + /// it was. It only settles the pending write it names. + Abandoned, } impl IndexOp { @@ -75,15 +83,21 @@ impl IndexOp { } } -/// One durable write: the operation, its `(bucket, key)`, and the writer's wall-clock -/// timestamp — the cross-writer LWW tiebreak, in **microseconds** so it is not coarser -/// than the clock a local write is stamped with (see [`wire_stamp`]). +/// One write-feed event: the operation, its `(bucket, key)`, and the writer's +/// wall-clock timestamp — the cross-writer LWW tiebreak, in **microseconds** so it is +/// not coarser than the clock a local write is stamped with (see [`wire_stamp`]). #[derive(Serialize, Deserialize)] pub(crate) struct IndexEvent { pub(crate) op: IndexOp, pub(crate) bucket: String, pub(crate) key: String, pub(crate) ts_us: u64, + /// The feed sequence of the [`IndexOp::Pending`] this event settles. + pub(crate) settles: Option, + /// The writer's oldest pending write still open when this event was published; + /// `None` when none was. A reader settles every older one it holds, which also + /// covers a settling event it lost to a gap. + pub(crate) floor: Option, } pub(crate) fn to_micros(ts: SystemTime) -> u64 { diff --git a/tests/it/coherence.rs b/tests/it/coherence.rs index d5b7845..eb6cdb3 100644 --- a/tests/it/coherence.rs +++ b/tests/it/coherence.rs @@ -238,14 +238,18 @@ async fn sustained_writes_keep_the_coherence_feed_moving() { timeouts, "the coherence feed arrested at sustained write {seq}" ); + // A strong write publishes its fence and then its outcome. assert_eq!( counter(&metrics_b, "feed_applied"), - applied + seq + 1, + applied + 2 * (seq + 1), "strong mode returned write {seq} before B applied it" ); } - assert_eq!(counter(&metrics_a, "feed_published"), published + WRITES); + assert_eq!( + counter(&metrics_a, "feed_published"), + published + 2 * WRITES + ); assert_eq!( counter(&metrics_a, "write_lease_lapses"), lapses, diff --git a/tests/it/common/faultnet.rs b/tests/it/common/faultnet.rs index ce5c602..435441f 100644 --- a/tests/it/common/faultnet.rs +++ b/tests/it/common/faultnet.rs @@ -139,6 +139,8 @@ impl Transport for FaultTransport { let inbound = Inbound { from: self.id.clone(), msg: msg.to_vec().into(), + // The fabric routes by identity and keeps no address book. + source: None, }; if delay.is_zero() { let _ = tx.send(inbound); diff --git a/tests/it/common/fleet.rs b/tests/it/common/fleet.rs index 79f1231..619d5d0 100644 --- a/tests/it/common/fleet.rs +++ b/tests/it/common/fleet.rs @@ -252,13 +252,22 @@ pub async fn mem_pod_node( let (name, peer) = (name.to_owned(), peer.to_owned()); pod.handle() .spawn(async move { - let sync = Arc::new(WriteSync::over_transport( - net.endpoint(groupnet::core::NodeId::new(name.as_str())), - &name, - &[peer.as_str()], - Consistency::Strong, - DEFAULT_LEASE_MS, - )); + let sync = Arc::new( + WriteSync::over_link( + groupnet::transport::link::BoundLink::new( + net.endpoint(groupnet::core::NodeId::new(name.as_str())), + groupnet::transport::link::LinkConfig::new(vec![ + groupnet::core::NodeId::new(peer.as_str()), + ]), + ), + &name, + &[peer.as_str()], + Consistency::Strong, + DEFAULT_LEASE_MS, + ) + .await + .expect("managed memory link starts"), + ); let proxy = proxy_over_with_metrics( &origin.counted_client(), CAP, diff --git a/tests/it/fleet_bootstrap.rs b/tests/it/fleet_bootstrap.rs index 26bbe1e..67b2818 100644 --- a/tests/it/fleet_bootstrap.rs +++ b/tests/it/fleet_bootstrap.rs @@ -2,6 +2,7 @@ use crate::common; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; @@ -718,9 +719,9 @@ async fn unreachable_donor_falls_back_to_origin_within_budget() { &broken_book, )) .with_recovery_config(RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 20_000, attempt_ms: 3_000, settle_ms: 3_000, @@ -1151,9 +1152,9 @@ const SLOW_READY_DEADLINE: Duration = Duration::from_secs(90); /// policy derived from it gives a builder a 4 s donor wait. fn stall_bounds() -> RecoveryConfig { RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 8_000, attempt_ms: 3_000, settle_ms: 3_000, @@ -1411,9 +1412,9 @@ async fn concurrent_start_follower_waits_out_a_production_paced_builder() { /// 30 s donor wait. fn short_episode() -> RecoveryConfig { RecoveryConfig { - max_members: 256, - max_member_bytes: 256, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(256).unwrap(), + max_member_bytes: NonZeroUsize::new(256).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 60_000, attempt_ms: 30_000, settle_ms: 3_000, diff --git a/tests/it/startup.rs b/tests/it/startup.rs index a5b0121..50ecb32 100644 --- a/tests/it/startup.rs +++ b/tests/it/startup.rs @@ -17,14 +17,15 @@ use s3cache::cache::proxy::CachingProxy; use s3cache::metrics::Metrics; use s3s::S3; use s3s::dto::GetObjectInput; +use std::num::{NonZeroU32, NonZeroUsize}; const CAP: usize = 1024 * 1024; fn short_recovery() -> RecoveryConfig { RecoveryConfig { - max_members: 16, - max_member_bytes: 128, - max_barrier_rounds: 4, + max_members: NonZeroUsize::new(16).unwrap(), + max_member_bytes: NonZeroUsize::new(128).unwrap(), + max_barrier_rounds: NonZeroU32::new(4).unwrap(), total_ms: 5_000, attempt_ms: 250, settle_ms: 2_000, diff --git a/tests/it/strong_rw.rs b/tests/it/strong_rw.rs index d5326ba..0f7beef 100644 --- a/tests/it/strong_rw.rs +++ b/tests/it/strong_rw.rs @@ -23,24 +23,24 @@ use crate::common; +mod workload; +use workload::{delete, get, head, list, put}; + use std::collections::BTreeMap; use std::fmt::Write as _; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; use bytes::Bytes; -use common::faultnet::{FaultNet, Link}; +use common::faultnet::FaultNet; use common::fleet::{CAP, Pod, SERIAL_SCAN, fleet_config_bound, free_tcp_port, serves_locally}; -use common::{Origin, WarmDir, body_blob, request}; +use common::{Origin, WarmDir}; use s3cache::cache::proxy::{CacheConfig, CachingProxy}; use s3cache::metrics::Metrics; use s3cache::sync::coherence::{Consistency, DEFAULT_LEASE_MS, WriteSync}; -use s3cache::tier::{buffer_body, open_warm}; -use s3s::dto::{ - DeleteObjectInput, ETag, ETagCondition, GetObjectInput, HeadObjectInput, ListObjectsV2Input, - PutObjectInput, Range, -}; -use s3s::{S3, S3Error, S3ErrorCode}; +use s3cache::tier::open_warm; +use s3s::S3ErrorCode; +use s3s::dto::{ETag, ETagCondition}; /// A hot tier smaller than the workload's bodies together, so most reads find /// their copy on the warm tier and have to prove it. @@ -240,13 +240,22 @@ impl Cluster { let proxy = pod .handle() .spawn(async move { - let sync = Arc::new(WriteSync::over_transport( - transport, - name, - &[peer], - Consistency::Strong, - DEFAULT_LEASE_MS, - )); + let sync = Arc::new( + WriteSync::over_link( + groupnet::transport::link::BoundLink::new( + transport, + groupnet::transport::link::LinkConfig::new(vec![ + groupnet::core::NodeId::new(peer), + ]), + ), + name, + &[peer], + Consistency::Strong, + DEFAULT_LEASE_MS, + ) + .await + .expect("managed fault link starts"), + ); let proxy = CachingProxy::new( &client, CacheConfig { @@ -814,6 +823,22 @@ impl Cluster { } } let _ = writeln!(text, "write failures: {failures:?}"); + let mut latencies: Vec = history + .writes + .iter() + .filter(|write| write.status == WriteStatus::Acked) + .map(|write| write.done - write.invoked) + .collect(); + latencies.sort(); + if let (Some(median), Some(p99)) = ( + latencies.get(latencies.len() / 2), + latencies.get(latencies.len() * 99 / 100), + ) { + let _ = writeln!( + text, + "acknowledged write latency: median {median:?}, p99 {p99:?}" + ); + } for violation in violations.iter().take(40) { let _ = writeln!(text, " [{}] {}", violation.class, violation.text); } @@ -907,642 +932,3 @@ impl Cond { } } } - -async fn put( - proxy: &CachingProxy, - bucket: &str, - key: &str, - payload: Bytes, - if_match: Option, - if_none_match: Option, -) -> Result, S3Error> { - let version = version_of(&payload).unwrap_or_default(); - // Every fourth version names no Content-Type: the proxy indexes it skeletally - // and keeps no copy, so its first HEAD and GET are the origin's to answer. - let content_type = - (!version.is_multiple_of(4)).then(|| "application/x-s3cache-check".to_owned()); - let input = PutObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - content_length: Some(i64::try_from(payload.len()).expect("small")), - body: Some(body_blob(payload)), - content_type, - metadata: Some([("version".to_owned(), version.to_string())].into()), - if_match, - if_none_match, - ..Default::default() - }; - let out = proxy.put_object(request(input)).await?.output; - Ok(out.e_tag.map(ETag::into_value)) -} - -async fn delete(proxy: &CachingProxy, bucket: &str, key: &str) -> Result<(), S3Error> { - let input = DeleteObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - ..Default::default() - }; - proxy.delete_object(request(input)).await?; - Ok(()) -} - -async fn get( - proxy: &CachingProxy, - bucket: &str, - key: &str, - range: Option<(u64, u64)>, -) -> Result { - let input = GetObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - range: range.map(|(first, last)| Range::Int { - first, - last: Some(last), - }), - ..Default::default() - }; - match proxy.get_object(request(input)).await { - Ok(mut out) => { - let blob = out.output.body.take().expect("a GET carries a body"); - let bytes = buffer_body(blob, usize::MAX).await.expect("readable body"); - Ok(version_of(&bytes).map_or(Value::Absent, Value::V)) - } - Err(error) if error.code() == &S3ErrorCode::NoSuchKey => Ok(Value::Absent), - Err(error) => Err(error), - } -} - -async fn head(proxy: &CachingProxy, bucket: &str, key: &str) -> Result { - let input = HeadObjectInput { - bucket: bucket.to_owned(), - key: key.to_owned(), - ..Default::default() - }; - match proxy.head_object(request(input)).await { - Ok(out) => { - let out = out.output; - Ok(Seen::Head { - etag: out.e_tag.map(ETag::into_value), - meta: out - .metadata - .as_ref() - .and_then(|meta| meta.get("version")) - .and_then(|version| version.parse().ok()), - length: out.content_length, - }) - } - Err(error) - if error.code() == &S3ErrorCode::NoSuchKey - || error.status_code() == Some(http::StatusCode::NOT_FOUND) => - { - Ok(Seen::Value(Value::Absent)) - } - Err(error) => Err(error), - } -} - -type Rows = BTreeMap, Option)>; - -async fn list(proxy: &CachingProxy, bucket: &str, prefix: &str) -> Result { - let out = proxy - .list_objects_v2(request(ListObjectsV2Input { - bucket: bucket.to_owned(), - prefix: Some(prefix.to_owned()), - ..Default::default() - })) - .await? - .output; - Ok(out - .contents - .into_iter() - .flatten() - .filter_map(|object| { - Some(( - object.key?, - (object.e_tag.map(ETag::into_value), object.size), - )) - }) - .collect()) -} - -/// A seeded splitmix64 stream per workload task. -struct Rng(u64); - -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - fn below(&mut self, bound: u64) -> u64 { - self.next() % bound - } - - fn node(&mut self) -> usize { - usize::from(self.next() % 2 == 1) - } -} - -/// The other node, unless it is down: then the same one. -fn other(cluster: &Cluster, via: usize) -> usize { - if cluster.node(1 - via).is_some() { - 1 - via - } else { - via - } -} - -/// A node that is up, preferring `via`. -fn up(cluster: &Cluster, via: usize) -> usize { - if cluster.node(via).is_some() { - via - } else { - 1 - via - } -} - -fn read_kind(rng: &mut Rng) -> ReadKind { - match rng.below(3) { - 0 => ReadKind::Get, - 1 => ReadKind::Range, - _ => ReadKind::Head, - } -} - -/// Overwrites of the pointer keys through alternating nodes, each followed by a -/// read through the other node. -async fn pointer_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut version = 0; - while Instant::now() < until { - version += 1; - let key = POINTERS[usize::try_from(version % 2).expect("small")]; - let via = up(&cluster, rng.node()); - if cluster.write(via, key, Value::V(version), Cond::None).await == WriteStatus::Acked { - let reader = other(&cluster, via); - cluster.read(reader, key, read_kind(&mut rng)).await; - if rng.below(4) == 0 { - cluster.list(reader, "ptr/").await; - } - } - } -} - -/// An `If-Match` chain: HEAD through one node, conditional overwrite through the -/// other, the way a log writer advances its pointer. -async fn cas_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut version = 1_000_000; - while Instant::now() < until { - let reader = up(&cluster, rng.node()); - let (bucket, key) = (cluster.bucket.clone(), CAS.to_owned()); - let invoked = Instant::now(); - let answer = cluster - .on(reader, move |proxy| async move { - head(&proxy, &bucket, &key).await - }) - .await - .and_then(Result::ok); - let done = Instant::now(); - let Some(current) = answer else { continue }; - cluster.history.lock().unwrap().reads.push(Read { - key: CAS.to_owned(), - op: "HEAD (CAS)", - via: reader, - invoked, - done, - seen: current.clone(), - }); - version += 1; - let cond = match current { - Seen::Head { - etag: Some(etag), .. - } => Cond::IfMatch(etag), - _ => Cond::IfNoneMatch, - }; - let writer = other(&cluster, reader); - cluster.write(writer, CAS, Value::V(version), cond).await; - } -} - -/// Log slots: create once (a contested duplicate through the other node must -/// lose), read and list through the other node, truncate old slots behind. -async fn slot_writer(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - let mut slot = 0_u64; - let mut version = 2_000_000; - while Instant::now() < until { - slot += 1; - let key = format!("log/{slot:04}"); - let via = up(&cluster, rng.node()); - version += 1; - if cluster - .write(via, &key, Value::V(version), Cond::IfNoneMatch) - .await - != WriteStatus::Acked - { - continue; - } - let reader = other(&cluster, via); - cluster.read(reader, &key, read_kind(&mut rng)).await; - version += 1; - cluster - .write(reader, &key, Value::V(version), Cond::IfNoneMatch) - .await; - cluster.list(reader, "log/").await; - if slot > 3 { - let old = format!("log/{:04}", slot - 3); - let deleter = up(&cluster, rng.node()); - if cluster - .write(deleter, &old, Value::Absent, Cond::None) - .await - == WriteStatus::Acked - { - let reader = other(&cluster, deleter); - cluster.read(reader, &old, read_kind(&mut rng)).await; - cluster.list(reader, "log/").await; - } - } - } -} - -/// Reads of every key through either node, at random. -async fn random_reader(cluster: Arc, seed: u64, until: Instant) { - let mut rng = Rng(seed); - while Instant::now() < until { - let via = up(&cluster, rng.node()); - let pick = rng.below(6); - if pick == 5 { - let prefix = ["ptr/", "cas/", "log/"][usize::try_from(rng.below(3)).expect("small")]; - cluster.list(via, prefix).await; - continue; - } - let key = match pick { - 0 | 1 => POINTERS[usize::try_from(pick).expect("small")].to_owned(), - 2 => CAS.to_owned(), - _ => { - let slots = cluster - .history - .lock() - .unwrap() - .writes - .iter() - .filter(|write| write.key.starts_with("log/")) - .map(|write| write.key.clone()) - .next_back(); - let Some(last) = slots else { continue }; - last - } - }; - cluster.read(via, &key, read_kind(&mut rng)).await; - tokio::time::sleep(Duration::from_millis(rng.below(15))).await; - } -} - -/// Run the workload for [`RUN`] while `faults` plays (it is handed the workload's -/// deadline, and may run workload of its own), heal, let the pair serve locally again, -/// read every key through both nodes, and assert the history honours the contract and -/// both nodes agree with the origin. -async fn scenario(label: &'static str, names: [&'static str; 2], seed: u64, faults: F) -where - F: FnOnce(Arc, Instant) -> Fut, - Fut: Future + Send + 'static, -{ - // `RUST_LOG` opts a run into the nodes' decisions, each under its pod's thread. - let _ = tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "off".into()), - ) - .with_thread_names(true) - .with_test_writer() - .try_init(); - let cluster = Cluster::new(label, names, seed).await; - cluster.note("workload starts"); - let until = Instant::now() + RUN; - let tasks = vec![ - tokio::spawn(pointer_writer(Arc::clone(&cluster), seed ^ 1, until)), - tokio::spawn(cas_writer(Arc::clone(&cluster), seed ^ 2, until)), - tokio::spawn(slot_writer(Arc::clone(&cluster), seed ^ 3, until)), - tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 4, until)), - tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 5, until)), - tokio::spawn(faults(Arc::clone(&cluster), until)), - ]; - for task in tasks { - task.await.expect("workload task"); - } - cluster.net.heal(); - cluster.origin.jitter_object_responses(Duration::ZERO); - cluster.origin.fail_applied_writes(0); - cluster.note("workload ends; faults healed"); - for index in 0..2 { - if cluster.node(index).is_none() { - cluster.boot(index).await; - } - } - cluster.await_local("the run").await; - cluster.settle_abandoned().await; - let keys: Vec = { - let history = cluster.history.lock().unwrap(); - let mut keys: Vec = history - .writes - .iter() - .map(|write| write.key.clone()) - .collect(); - keys.sort(); - keys.dedup(); - keys - }; - let mut prefixes: Vec = keys - .iter() - .filter_map(|key| key.split_once('/').map(|(head, _)| format!("{head}/"))) - .collect(); - prefixes.dedup(); - for via in 0..2 { - for key in &keys { - for kind in [ReadKind::Get, ReadKind::Range, ReadKind::Head] { - cluster.read(via, key, kind).await; - } - } - for prefix in &prefixes { - cluster.list(via, prefix).await; - } - } - let mut violations = cluster.violations(); - violations.extend(cluster.divergence(&keys).await); - let report = cluster.report(&violations); - println!("{report}"); - assert!(violations.is_empty(), "{report}"); -} - -/// A cut link for longer than the lease: the writer's wait ends on a lapse. -fn cut() -> Link { - Link { - cut: true, - ..Link::default() - } -} - -/// How long a cut must last for each side to reap the other and then keep serving -/// past a stalled write's whole wait: the binary tunes `dead_timeout_ms` to the lease -/// `D` and reaps a member `2 × dead_timeout_ms` past its `Dead` verdict, which follows -/// the silence by a detection window plus the suspect timeout (well inside one more -/// `D`), and a write waits at most `D` plus a second of slack. -const PAST_THE_REAP_HORIZON: Duration = Duration::from_millis(5 * DEFAULT_LEASE_MS); - -/// Steady state, with origin answers arriving out of order. -#[tokio::test(flavor = "multi_thread")] -async fn steady_state_with_origin_jitter() { - scenario( - "rw-steady", - ["rw-steady-a", "rw-steady-b"], - 11, - |cluster, _| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(40)); - cluster.note("origin jitter 0-40ms"); - }, - ) - .await; -} - -/// Gossip delayed and lossy in both directions, below the lease. -#[tokio::test(flavor = "multi_thread")] -async fn feed_delay_and_gossip_loss() { - scenario( - "rw-lossy", - ["rw-lossy-a", "rw-lossy-b"], - 12, - |cluster, _| async move { - let [a, b] = cluster.names; - cluster.net.both( - a, - b, - Link { - delay: Duration::from_millis(150), - loss_per_mille: 300, - ..Link::default() - }, - ); - cluster.note("both directions: 150ms delay, 30% loss"); - cluster - .origin - .jitter_object_responses(Duration::from_millis(20)); - }, - ) - .await; -} - -/// Full partitions longer than the lease, then heals: writes complete on lapses, -/// and the lapsed side recovers. -#[tokio::test(flavor = "multi_thread")] -async fn partitions_longer_than_the_lease() { - scenario( - "rw-part", - ["rw-part-a", "rw-part-b"], - 13, - |cluster, _| async move { - let [a, b] = cluster.names; - for _ in 0..2 { - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.both(a, b, cut()); - cluster.note("partition a<->b"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - } - }, - ) - .await; -} - -/// One direction cut at a time, longer than the lease. -#[tokio::test(flavor = "multi_thread")] -async fn asymmetric_partitions() { - scenario( - "rw-asym", - ["rw-asym-a", "rw-asym-b"], - 14, - |cluster, _| async move { - let [a, b] = cluster.names; - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.set(a, b, cut()); - cluster.note("cut a->b"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - tokio::time::sleep(Duration::from_secs(2)).await; - cluster.net.set(b, a, cut()); - cluster.note("cut b->a"); - tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; - cluster.net.heal(); - cluster.note("heal"); - }, - ) - .await; -} - -/// One direction cut for longer than the reap horizon: the reader that hears nothing -/// from the writer stops counting it as a granter while the writer still counts the -/// reader — the asymmetric partition behind a stalled acknowledgement. -#[tokio::test(flavor = "multi_thread")] -async fn an_asymmetric_partition_past_the_reap_horizon() { - scenario( - "rw-asym-reap", - ["rw-asym-reap-a", "rw-asym-reap-b"], - 21, - |cluster, _| async move { - let [a, b] = cluster.names; - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.net.set(a, b, cut()); - cluster.note("cut a->b past the reap horizon"); - tokio::time::sleep(PAST_THE_REAP_HORIZON).await; - cluster.net.heal(); - cluster.note("heal"); - }, - ) - .await; -} - -/// A pod whose only worker stalls past the lease: its lease lapses under writes. -#[tokio::test(flavor = "multi_thread")] -async fn a_stalled_pod_lapses() { - scenario( - "rw-stall", - ["rw-stall-a", "rw-stall-b"], - 15, - |cluster, _| async move { - for index in [1, 0] { - tokio::time::sleep(Duration::from_secs(3)).await; - if let Some(pod) = cluster.pods.lock().unwrap()[index].as_ref() { - pod.stall(LEASE + Duration::from_millis(800)); - } - cluster.note(format!("{} stalls", cluster.names[index])); - } - }, - ) - .await; -} - -/// A pod crashes mid-workload and comes back on its warm tier. -#[tokio::test(flavor = "multi_thread")] -async fn a_crashed_pod_restarts_on_its_warm_tier() { - scenario( - "rw-crash", - ["rw-crash-a", "rw-crash-b"], - 16, - |cluster, _| async move { - tokio::time::sleep(Duration::from_secs(3)).await; - cluster.crash(1).await; - tokio::time::sleep(Duration::from_secs(3)).await; - cluster.boot(1).await; - }, - ) - .await; -} - -/// A rolling update: each pod in turn takes the binary's planned stop and -/// restarts on its warm tier while the other serves. -#[tokio::test(flavor = "multi_thread")] -async fn a_rolling_update() { - scenario( - "rw-roll", - ["rw-roll-a", "rw-roll-b"], - 17, - |cluster, _| async move { - for index in [1, 0] { - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.planned_stop(index).await; - tokio::time::sleep(Duration::from_secs(1)).await; - cluster.boot(index).await; - } - }, - ) - .await; -} - -/// The origin commits some writes and then fails their responses. -#[tokio::test(flavor = "multi_thread")] -async fn ambiguous_write_outcomes() { - scenario( - "rw-ambig", - ["rw-ambig-a", "rw-ambig-b"], - 18, - |cluster, _| async move { - cluster.origin.fail_applied_writes(150); - cluster - .origin - .jitter_object_responses(Duration::from_millis(20)); - cluster.note("15% of applied writes answered 500; jitter 0-20ms"); - }, - ) - .await; -} - -/// Clients that hang up mid-write while the origin answers late: an abandoned DELETE -/// or PUT still lands in every node's view of the key. -#[tokio::test(flavor = "multi_thread")] -async fn clients_that_hang_up_mid_write() { - scenario( - "rw-hangup", - ["rw-hangup-a", "rw-hangup-b"], - 19, - |cluster, until| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(60)); - cluster.note("origin jitter 0-60ms; clients give up after 0-40ms"); - let mut rng = Rng(19); - let mut n = 3_000_000; - while Instant::now() < until { - n += 1; - let key = format!("cancel/{n}"); - let via = up(&cluster, rng.node()); - if cluster.write(via, &key, Value::V(n), Cond::None).await != WriteStatus::Acked { - continue; - } - let patience = Duration::from_millis(rng.below(40)); - let other = other(&cluster, via); - cluster - .write_within(other, &key, Value::Absent, Cond::None, Some(patience)) - .await; - } - }, - ) - .await; -} - -/// Two clients overwrite one key through different nodes at once: the origin -/// decides the order, and both nodes must end up answering what it kept. -#[tokio::test(flavor = "multi_thread")] -async fn concurrent_overwrites_through_both_nodes() { - scenario( - "rw-race", - ["rw-race-a", "rw-race-b"], - 20, - |cluster, until| async move { - cluster - .origin - .jitter_object_responses(Duration::from_millis(30)); - let racers = (0..2).map(|via| { - let cluster = Arc::clone(&cluster); - tokio::spawn(async move { - let mut n = 4_000_000 + via as u64 * 1_000_000; - while Instant::now() < until { - n += 1; - cluster - .write(up(&cluster, via), "race/k", Value::V(n), Cond::None) - .await; - } - }) - }); - for racer in racers.collect::>() { - racer.await.expect("racer"); - } - }, - ) - .await; -} diff --git a/tests/it/strong_rw/workload.rs b/tests/it/strong_rw/workload.rs new file mode 100644 index 0000000..fa1c352 --- /dev/null +++ b/tests/it/strong_rw/workload.rs @@ -0,0 +1,662 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::common::faultnet::Link; +use crate::common::{body_blob, request}; +use bytes::Bytes; +use s3cache::cache::proxy::CachingProxy; +use s3cache::sync::coherence::DEFAULT_LEASE_MS; +use s3cache::tier::buffer_body; +use s3s::dto::{ + DeleteObjectInput, ETag, ETagCondition, GetObjectInput, HeadObjectInput, ListObjectsV2Input, + PutObjectInput, Range, +}; +use s3s::{S3, S3Error, S3ErrorCode}; + +use super::{ + CAS, Cluster, Cond, LEASE, POINTERS, RUN, Read, ReadKind, Seen, Value, WriteStatus, version_of, +}; + +pub(super) async fn put( + proxy: &CachingProxy, + bucket: &str, + key: &str, + payload: Bytes, + if_match: Option, + if_none_match: Option, +) -> Result, S3Error> { + let version = version_of(&payload).unwrap_or_default(); + // Every fourth version names no Content-Type: the proxy indexes it skeletally + // and keeps no copy, so its first HEAD and GET are the origin's to answer. + let content_type = + (!version.is_multiple_of(4)).then(|| "application/x-s3cache-check".to_owned()); + let input = PutObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + content_length: Some(i64::try_from(payload.len()).expect("small")), + body: Some(body_blob(payload)), + content_type, + metadata: Some([("version".to_owned(), version.to_string())].into()), + if_match, + if_none_match, + ..Default::default() + }; + let out = proxy.put_object(request(input)).await?.output; + Ok(out.e_tag.map(ETag::into_value)) +} + +pub(super) async fn delete(proxy: &CachingProxy, bucket: &str, key: &str) -> Result<(), S3Error> { + let input = DeleteObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + ..Default::default() + }; + proxy.delete_object(request(input)).await?; + Ok(()) +} + +pub(super) async fn get( + proxy: &CachingProxy, + bucket: &str, + key: &str, + range: Option<(u64, u64)>, +) -> Result { + let input = GetObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + range: range.map(|(first, last)| Range::Int { + first, + last: Some(last), + }), + ..Default::default() + }; + match proxy.get_object(request(input)).await { + Ok(mut out) => { + let blob = out.output.body.take().expect("a GET carries a body"); + let bytes = buffer_body(blob, usize::MAX).await.expect("readable body"); + Ok(version_of(&bytes).map_or(Value::Absent, Value::V)) + } + Err(error) if error.code() == &S3ErrorCode::NoSuchKey => Ok(Value::Absent), + Err(error) => Err(error), + } +} + +pub(super) async fn head(proxy: &CachingProxy, bucket: &str, key: &str) -> Result { + let input = HeadObjectInput { + bucket: bucket.to_owned(), + key: key.to_owned(), + ..Default::default() + }; + match proxy.head_object(request(input)).await { + Ok(out) => { + let out = out.output; + Ok(Seen::Head { + etag: out.e_tag.map(ETag::into_value), + meta: out + .metadata + .as_ref() + .and_then(|meta| meta.get("version")) + .and_then(|version| version.parse().ok()), + length: out.content_length, + }) + } + Err(error) + if error.code() == &S3ErrorCode::NoSuchKey + || error.status_code() == Some(http::StatusCode::NOT_FOUND) => + { + Ok(Seen::Value(Value::Absent)) + } + Err(error) => Err(error), + } +} + +pub(super) type Rows = BTreeMap, Option)>; + +pub(super) async fn list( + proxy: &CachingProxy, + bucket: &str, + prefix: &str, +) -> Result { + let out = proxy + .list_objects_v2(request(ListObjectsV2Input { + bucket: bucket.to_owned(), + prefix: Some(prefix.to_owned()), + ..Default::default() + })) + .await? + .output; + Ok(out + .contents + .into_iter() + .flatten() + .filter_map(|object| { + Some(( + object.key?, + (object.e_tag.map(ETag::into_value), object.size), + )) + }) + .collect()) +} + +/// A seeded splitmix64 stream per workload task. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn node(&mut self) -> usize { + usize::from(self.next() % 2 == 1) + } +} + +/// The other node, unless it is down: then the same one. +fn other(cluster: &Cluster, via: usize) -> usize { + if cluster.node(1 - via).is_some() { + 1 - via + } else { + via + } +} + +/// A node that is up, preferring `via`. +fn up(cluster: &Cluster, via: usize) -> usize { + if cluster.node(via).is_some() { + via + } else { + 1 - via + } +} + +fn read_kind(rng: &mut Rng) -> ReadKind { + match rng.below(3) { + 0 => ReadKind::Get, + 1 => ReadKind::Range, + _ => ReadKind::Head, + } +} + +/// Overwrites of the pointer keys through alternating nodes, each followed by a +/// read through the other node. +async fn pointer_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut version = 0; + while Instant::now() < until { + version += 1; + let key = POINTERS[usize::try_from(version % 2).expect("small")]; + let via = up(&cluster, rng.node()); + if cluster.write(via, key, Value::V(version), Cond::None).await == WriteStatus::Acked { + let reader = other(&cluster, via); + cluster.read(reader, key, read_kind(&mut rng)).await; + if rng.below(4) == 0 { + cluster.list(reader, "ptr/").await; + } + } + } +} + +/// An `If-Match` chain: HEAD through one node, conditional overwrite through the +/// other, the way a log writer advances its pointer. +async fn cas_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut version = 1_000_000; + while Instant::now() < until { + let reader = up(&cluster, rng.node()); + let (bucket, key) = (cluster.bucket.clone(), CAS.to_owned()); + let invoked = Instant::now(); + let answer = cluster + .on(reader, move |proxy| async move { + head(&proxy, &bucket, &key).await + }) + .await + .and_then(Result::ok); + let done = Instant::now(); + let Some(current) = answer else { continue }; + cluster.history.lock().unwrap().reads.push(Read { + key: CAS.to_owned(), + op: "HEAD (CAS)", + via: reader, + invoked, + done, + seen: current.clone(), + }); + version += 1; + let cond = match current { + Seen::Head { + etag: Some(etag), .. + } => Cond::IfMatch(etag), + _ => Cond::IfNoneMatch, + }; + let writer = other(&cluster, reader); + cluster.write(writer, CAS, Value::V(version), cond).await; + } +} + +/// Log slots: create once (a contested duplicate through the other node must +/// lose), read and list through the other node, truncate old slots behind. +async fn slot_writer(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + let mut slot = 0_u64; + let mut version = 2_000_000; + while Instant::now() < until { + slot += 1; + let key = format!("log/{slot:04}"); + let via = up(&cluster, rng.node()); + version += 1; + if cluster + .write(via, &key, Value::V(version), Cond::IfNoneMatch) + .await + != WriteStatus::Acked + { + continue; + } + let reader = other(&cluster, via); + cluster.read(reader, &key, read_kind(&mut rng)).await; + version += 1; + cluster + .write(reader, &key, Value::V(version), Cond::IfNoneMatch) + .await; + cluster.list(reader, "log/").await; + if slot > 3 { + let old = format!("log/{:04}", slot - 3); + let deleter = up(&cluster, rng.node()); + if cluster + .write(deleter, &old, Value::Absent, Cond::None) + .await + == WriteStatus::Acked + { + let reader = other(&cluster, deleter); + cluster.read(reader, &old, read_kind(&mut rng)).await; + cluster.list(reader, "log/").await; + } + } + } +} + +/// Reads of every key through either node, at random. +async fn random_reader(cluster: Arc, seed: u64, until: Instant) { + let mut rng = Rng(seed); + while Instant::now() < until { + let via = up(&cluster, rng.node()); + let pick = rng.below(6); + if pick == 5 { + let prefix = ["ptr/", "cas/", "log/"][usize::try_from(rng.below(3)).expect("small")]; + cluster.list(via, prefix).await; + continue; + } + let key = match pick { + 0 | 1 => POINTERS[usize::try_from(pick).expect("small")].to_owned(), + 2 => CAS.to_owned(), + _ => { + let slots = cluster + .history + .lock() + .unwrap() + .writes + .iter() + .filter(|write| write.key.starts_with("log/")) + .map(|write| write.key.clone()) + .next_back(); + let Some(last) = slots else { continue }; + last + } + }; + cluster.read(via, &key, read_kind(&mut rng)).await; + tokio::time::sleep(Duration::from_millis(rng.below(15))).await; + } +} + +/// Run the workload for [`RUN`] while `faults` plays (it is handed the workload's +/// deadline, and may run workload of its own), heal, let the pair serve locally again, +/// read every key through both nodes, and assert the history honours the contract and +/// both nodes agree with the origin. +async fn scenario(label: &'static str, names: [&'static str; 2], seed: u64, faults: F) +where + F: FnOnce(Arc, Instant) -> Fut, + Fut: Future + Send + 'static, +{ + // `RUST_LOG` opts a run into the nodes' decisions, each under its pod's thread. + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "off".into()), + ) + .with_thread_names(true) + .with_test_writer() + .try_init(); + let cluster = Cluster::new(label, names, seed).await; + cluster.note("workload starts"); + let until = Instant::now() + RUN; + let tasks = vec![ + tokio::spawn(pointer_writer(Arc::clone(&cluster), seed ^ 1, until)), + tokio::spawn(cas_writer(Arc::clone(&cluster), seed ^ 2, until)), + tokio::spawn(slot_writer(Arc::clone(&cluster), seed ^ 3, until)), + tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 4, until)), + tokio::spawn(random_reader(Arc::clone(&cluster), seed ^ 5, until)), + tokio::spawn(faults(Arc::clone(&cluster), until)), + ]; + for task in tasks { + task.await.expect("workload task"); + } + cluster.net.heal(); + cluster.origin.jitter_object_responses(Duration::ZERO); + cluster.origin.fail_applied_writes(0); + cluster.note("workload ends; faults healed"); + for index in 0..2 { + if cluster.node(index).is_none() { + cluster.boot(index).await; + } + } + cluster.await_local("the run").await; + cluster.settle_abandoned().await; + let keys: Vec = { + let history = cluster.history.lock().unwrap(); + let mut keys: Vec = history + .writes + .iter() + .map(|write| write.key.clone()) + .collect(); + keys.sort(); + keys.dedup(); + keys + }; + let mut prefixes: Vec = keys + .iter() + .filter_map(|key| key.split_once('/').map(|(head, _)| format!("{head}/"))) + .collect(); + prefixes.dedup(); + for via in 0..2 { + for key in &keys { + for kind in [ReadKind::Get, ReadKind::Range, ReadKind::Head] { + cluster.read(via, key, kind).await; + } + } + for prefix in &prefixes { + cluster.list(via, prefix).await; + } + } + let mut violations = cluster.violations(); + violations.extend(cluster.divergence(&keys).await); + let report = cluster.report(&violations); + println!("{report}"); + assert!(violations.is_empty(), "{report}"); +} + +/// A cut link for longer than the lease: the writer's wait ends on a lapse. +fn cut() -> Link { + Link { + cut: true, + ..Link::default() + } +} + +/// How long a cut must last for each side to reap the other and then keep serving +/// past a stalled write's whole wait: the binary tunes `dead_timeout_ms` to the lease +/// `D` and reaps a member `2 × dead_timeout_ms` past its `Dead` verdict, which follows +/// the silence by a detection window plus the suspect timeout (well inside one more +/// `D`), and a write waits at most `D` plus a second of slack. +const PAST_THE_REAP_HORIZON: Duration = Duration::from_millis(5 * DEFAULT_LEASE_MS); + +/// Steady state, with origin answers arriving out of order. +#[tokio::test(flavor = "multi_thread")] +async fn steady_state_with_origin_jitter() { + scenario( + "rw-steady", + ["rw-steady-a", "rw-steady-b"], + 11, + |cluster, _| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(40)); + cluster.note("origin jitter 0-40ms"); + }, + ) + .await; +} + +/// Gossip delayed and lossy in both directions, below the lease. +#[tokio::test(flavor = "multi_thread")] +async fn feed_delay_and_gossip_loss() { + scenario( + "rw-lossy", + ["rw-lossy-a", "rw-lossy-b"], + 12, + |cluster, _| async move { + let [a, b] = cluster.names; + cluster.net.both( + a, + b, + Link { + delay: Duration::from_millis(150), + loss_per_mille: 300, + ..Link::default() + }, + ); + cluster.note("both directions: 150ms delay, 30% loss"); + cluster + .origin + .jitter_object_responses(Duration::from_millis(20)); + }, + ) + .await; +} + +/// Full partitions longer than the lease, then heals: writes complete on lapses, +/// and the lapsed side recovers. +#[tokio::test(flavor = "multi_thread")] +async fn partitions_longer_than_the_lease() { + scenario( + "rw-part", + ["rw-part-a", "rw-part-b"], + 13, + |cluster, _| async move { + let [a, b] = cluster.names; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.both(a, b, cut()); + cluster.note("partition a<->b"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + } + }, + ) + .await; +} + +/// One direction cut at a time, longer than the lease. +#[tokio::test(flavor = "multi_thread")] +async fn asymmetric_partitions() { + scenario( + "rw-asym", + ["rw-asym-a", "rw-asym-b"], + 14, + |cluster, _| async move { + let [a, b] = cluster.names; + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.set(a, b, cut()); + cluster.note("cut a->b"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + tokio::time::sleep(Duration::from_secs(2)).await; + cluster.net.set(b, a, cut()); + cluster.note("cut b->a"); + tokio::time::sleep(LEASE + Duration::from_millis(1500)).await; + cluster.net.heal(); + cluster.note("heal"); + }, + ) + .await; +} + +/// One direction cut for longer than the reap horizon: the reader that hears nothing +/// from the writer stops counting it as a granter while the writer still counts the +/// reader — the asymmetric partition behind a stalled acknowledgement. +#[tokio::test(flavor = "multi_thread")] +async fn an_asymmetric_partition_past_the_reap_horizon() { + scenario( + "rw-asym-reap", + ["rw-asym-reap-a", "rw-asym-reap-b"], + 21, + |cluster, _| async move { + let [a, b] = cluster.names; + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.net.set(a, b, cut()); + cluster.note("cut a->b past the reap horizon"); + tokio::time::sleep(PAST_THE_REAP_HORIZON).await; + cluster.net.heal(); + cluster.note("heal"); + }, + ) + .await; +} + +/// A pod whose only worker stalls past the lease: its lease lapses under writes. +#[tokio::test(flavor = "multi_thread")] +async fn a_stalled_pod_lapses() { + scenario( + "rw-stall", + ["rw-stall-a", "rw-stall-b"], + 15, + |cluster, _| async move { + for index in [1, 0] { + tokio::time::sleep(Duration::from_secs(3)).await; + if let Some(pod) = cluster.pods.lock().unwrap()[index].as_ref() { + pod.stall(LEASE + Duration::from_millis(800)); + } + cluster.note(format!("{} stalls", cluster.names[index])); + } + }, + ) + .await; +} + +/// A pod crashes mid-workload and comes back on its warm tier. +#[tokio::test(flavor = "multi_thread")] +async fn a_crashed_pod_restarts_on_its_warm_tier() { + scenario( + "rw-crash", + ["rw-crash-a", "rw-crash-b"], + 16, + |cluster, _| async move { + tokio::time::sleep(Duration::from_secs(3)).await; + cluster.crash(1).await; + tokio::time::sleep(Duration::from_secs(3)).await; + cluster.boot(1).await; + }, + ) + .await; +} + +/// A rolling update: each pod in turn takes the binary's planned stop and +/// restarts on its warm tier while the other serves. +#[tokio::test(flavor = "multi_thread")] +async fn a_rolling_update() { + scenario( + "rw-roll", + ["rw-roll-a", "rw-roll-b"], + 17, + |cluster, _| async move { + for index in [1, 0] { + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.planned_stop(index).await; + tokio::time::sleep(Duration::from_secs(1)).await; + cluster.boot(index).await; + } + }, + ) + .await; +} + +/// The origin commits some writes and then fails their responses. +#[tokio::test(flavor = "multi_thread")] +async fn ambiguous_write_outcomes() { + scenario( + "rw-ambig", + ["rw-ambig-a", "rw-ambig-b"], + 18, + |cluster, _| async move { + cluster.origin.fail_applied_writes(150); + cluster + .origin + .jitter_object_responses(Duration::from_millis(20)); + cluster.note("15% of applied writes answered 500; jitter 0-20ms"); + }, + ) + .await; +} + +/// Clients that hang up mid-write while the origin answers late: an abandoned DELETE +/// or PUT still lands in every node's view of the key. +#[tokio::test(flavor = "multi_thread")] +async fn clients_that_hang_up_mid_write() { + scenario( + "rw-hangup", + ["rw-hangup-a", "rw-hangup-b"], + 19, + |cluster, until| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(60)); + cluster.note("origin jitter 0-60ms; clients give up after 0-40ms"); + let mut rng = Rng(19); + let mut n = 3_000_000; + while Instant::now() < until { + n += 1; + let key = format!("cancel/{n}"); + let via = up(&cluster, rng.node()); + if cluster.write(via, &key, Value::V(n), Cond::None).await != WriteStatus::Acked { + continue; + } + let patience = Duration::from_millis(rng.below(40)); + let other = other(&cluster, via); + cluster + .write_within(other, &key, Value::Absent, Cond::None, Some(patience)) + .await; + } + }, + ) + .await; +} + +/// Two clients overwrite one key through different nodes at once: the origin +/// decides the order, and both nodes must end up answering what it kept. +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_overwrites_through_both_nodes() { + scenario( + "rw-race", + ["rw-race-a", "rw-race-b"], + 20, + |cluster, until| async move { + cluster + .origin + .jitter_object_responses(Duration::from_millis(30)); + let racers = (0..2).map(|via| { + let cluster = Arc::clone(&cluster); + tokio::spawn(async move { + let mut n = 4_000_000 + via as u64 * 1_000_000; + while Instant::now() < until { + n += 1; + cluster + .write(up(&cluster, via), "race/k", Value::V(n), Cond::None) + .await; + } + }) + }); + for racer in racers.collect::>() { + racer.await.expect("racer"); + } + }, + ) + .await; +}