From 208545d530ad02f80723ebf86869909aae19712b Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Thu, 3 Sep 2026 04:02:35 +0000 Subject: [PATCH 1/2] Optimizations 1. add support for pgext to have multiple background workers for parallel oracle requests 2. resolver pool between decoder pools & inserter pools to improve concurrency 3. remove bootstrap mutex, tar entries processed in parallel 4. skip decoding pages in bootstrap for unused tables 5. threads track local counters before adding to atomics, reducing contention 6. reduce copies by using slabs, batch tuples to mpsc to reduce contention 7. OracleCell::Literal can be resolved without oracle (TODO: investigate if this can be simplified) --- Cargo.toml | 11 + benches/bootstrap_pump.rs | 481 +++++++++++ benches/oracle_batch.rs | 392 +++++++++ pgext/worker.c | 209 +++-- plans/GLOSSARY.md | 27 +- plans/bootstrap.md | 133 +++- plans/emitter.md | 27 +- plans/oracle.md | 58 +- src/backfill/backfill_bootstrap.rs | 209 +++-- src/backfill/backup_backfill.rs | 99 +-- src/backfill/backup_page_walk.rs | 876 +++++++++++++-------- src/backfill/backup_sink.rs | 278 +++---- src/backfill/backup_source.rs | 280 ++++--- src/backfill/backup_source_direct.rs | 30 +- src/backfill/backup_source_object_store.rs | 58 +- src/backfill/bootstrap_oracle.rs | 51 +- src/backfill/copy_backfill.rs | 55 +- src/bin/stream.rs | 258 ++++-- src/catalog/shadow.rs | 10 +- src/emit/ch_emitter.rs | 82 ++ src/emit/pipeline/bootstrap.rs | 266 ++++--- src/emit/pipeline/inserter.rs | 111 +-- src/emit/pipeline/mod.rs | 1 + src/emit/pipeline/resolver.rs | 384 +++++++++ src/emit/pipeline/tail.rs | 27 +- src/ops/bridge.rs | 229 ++++-- src/ops/metrics.rs | 171 +++- src/ops/oracle.rs | 37 +- tests/bootstrap_direct_e2e.rs | 60 +- tests/bootstrap_object_store_e2e.rs | 55 +- tests/bootstrap_pipeline_ch.rs | 6 +- tests/bridge.rs | 134 +++- tests/common/inproc_harness.rs | 2 +- tests/oracle.rs | 12 +- tests/oracle_types_e2e.rs | 2 +- tests/runtime_config_e2e.rs | 2 +- tests/shadow_catalog.rs | 2 +- tests/xact_buffer.rs | 1 + 38 files changed, 3864 insertions(+), 1262 deletions(-) create mode 100644 benches/bootstrap_pump.rs create mode 100644 benches/oracle_batch.rs create mode 100644 src/emit/pipeline/resolver.rs diff --git a/Cargo.toml b/Cargo.toml index 85eece44..f5ad7c3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,17 @@ opentelemetry-otlp = { version = "0.32", default-features = false, features = [" tracing-opentelemetry = { version = "0.33", default-features = false } mimalloc = "0.1.52" +# `harness = false` throughout: these take shape flags rather than libtest +# filters. Bootstrap pump + drain needs no PG and no CH; oracle batch needs +# a shadow PG for its roundtrip stage only +[[bench]] +name = "bootstrap_pump" +harness = false + +[[bench]] +name = "oracle_batch" +harness = false + [dev-dependencies] tempfile = "3" # Enable paused time for batch deadline tests diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs new file mode 100644 index 00000000..cf837a7a --- /dev/null +++ b/benches/bootstrap_pump.rs @@ -0,0 +1,481 @@ +//! Bootstrap pump + drain throughput, no Postgres and no ClickHouse. +//! +//! Two stages the greenfield initial load is made of, measured apart: +//! +//! - `pump`: in-memory BASE_BACKUP tars → `MultiplexSink(DiskLanderSink, +//! PageWalkSink)` → tuple channel → null drain. Covers tar parse, page +//! framing, heap decode and the tuple-channel hop, under `--parallelism` +//! concurrent tar parts so shared-sink contention shows up. +//! - `drain`: synthetic `BackfillTuple`s → `pipeline::bootstrap::drain` → +//! metrics-only tail. Covers routing plus the `BatcherMsg` hop. +//! +//! ```text +//! cargo bench --bench bootstrap_pump -- \ +//! --stage pump --parts 8 --segments-per-part 4 --pages 1024 --parallelism 4 +//! ``` +//! +//! Must run on mimalloc, the allocator `walshadow-stream` sets: the walk +//! produces decoded values on one thread and the drain frees them on +//! another, and glibc's shared arena serializes on that pattern hard +//! enough to invert the conclusion — there the parallel pump measures +//! slower than the serial one it replaced. +//! +//! `harness = false`, so nextest lists it by answering `--list`. + +// Matches `walshadow-stream`: the walk allocates decoded values on one +// thread and the drain frees them on another, and glibc's shared arena +// serializes on that pattern. A bench on a different allocator measures a +// different program +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use walrus::pg::replication::base_backup::Tablespace; +use walrus::pg::walparser::RelFileNode; + +use walshadow::backfill::backup_source::{ + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, +}; +use walshadow::backfill_bootstrap::{BootstrapConfig, spawn_greenfield_bootstrap}; +use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, PAGE_BYTES}; +use walshadow::heap_decoder::ColumnValue; +use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; +use walshadow::pipeline::bootstrap; +use walshadow::pos::{EmitterAck, Monotone}; +use walshadow::schema::{INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent}; +use walshadow::toast::ToastResolver; + +const DB_NODE: u32 = 5; +const FIRST_FILENODE: u32 = 16400; +const START_LSN: u64 = 0x5000_0000; +/// `HeapTupleHeaderData` + pad to the 8-byte-aligned user-data offset +const TUPLE_HEADER: usize = 24; +const SIZE_OF_PAGE_HEADER: usize = 24; +const SIZE_OF_ITEM_ID: usize = 4; +const LP_NORMAL: u32 = 1; + +#[derive(Clone, Copy)] +struct Shape { + /// Concurrent tar parts, as an object-store backup is partitioned + parts: usize, + /// Relation segments per part, one tar entry each + segments_per_part: usize, + pages_per_segment: usize, + tuples_per_page: usize, + /// int4 columns per tuple + columns: usize, + parallelism: usize, +} + +impl Default for Shape { + fn default() -> Self { + Self { + parts: 8, + segments_per_part: 4, + pages_per_segment: 256, + tuples_per_page: 100, + columns: 8, + parallelism: 4, + } + } +} + +impl Shape { + fn segments(self) -> usize { + self.parts * self.segments_per_part + } + + fn tuples(self) -> u64 { + (self.segments() * self.pages_per_segment * self.tuples_per_page) as u64 + } + + fn bytes(self) -> u64 { + (self.segments() * self.pages_per_segment * PAGE_BYTES) as u64 + } +} + +fn descriptor(filenode: u32, columns: usize) -> Arc { + RelDescriptor { + rfn: RelFileNode { + spc_node: 1663, + db_node: DB_NODE, + rel_node: filenode, + }, + oid: filenode, + toast_oid: 0, + namespace_oid: 2200, + rel_name: RelName::new("public", &format!("t{filenode}")), + kind: 'r', + persistence: 'p', + replident: ReplIdent::Default { pk_attnums: None }, + attributes: (0..columns) + .map(|i| RelAttr { + attnum: i as i16 + 1, + name: format!("c{i}"), + type_oid: INT4OID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "int4".into(), + type_byval: true, + type_len: 4, + type_align: 'i', + type_storage: 'p', + missing_text: None, + }) + .collect(), + } + .into() +} + +fn mapping(filenode: u32, columns: usize) -> TableMapping { + TableMapping { + target: TableTarget::new("bench", &format!("t{filenode}")), + columns: (0..columns) + .map(|i| ColumnMapping { + src_attnum: i as i16 + 1, + target_name: format!("c{i}"), + target_type: "Int32".into(), + }) + .collect(), + } +} + +/// One 8 KiB heap page carrying `tuples` live int4-only tuples, laid out +/// as PG writes them: line pointers up from the header, tuple bodies down +/// from the page end. +fn heap_page(tuples: usize, columns: usize, seed: i32) -> Vec { + let body = TUPLE_HEADER + 4 * columns; + let mut page = vec![0u8; PAGE_BYTES]; + let mut written = 0usize; + for i in 0..tuples { + let off = PAGE_BYTES - (i + 1) * body; + if off < SIZE_OF_PAGE_HEADER + (i + 1) * SIZE_OF_ITEM_ID { + break; + } + let xmin = 100u32 + i as u32; + page[off..off + 4].copy_from_slice(&xmin.to_le_bytes()); + page[off + 18..off + 20].copy_from_slice(&(columns as u16).to_le_bytes()); + page[off + 20..off + 22].copy_from_slice(&0u16.to_le_bytes()); + page[off + 22] = TUPLE_HEADER as u8; + for c in 0..columns { + let at = off + TUPLE_HEADER + c * 4; + let v = seed + (i as i32) * 31 + c as i32; + page[at..at + 4].copy_from_slice(&v.to_le_bytes()); + } + let slot = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; + let raw = ((off as u32) & 0x7FFF) | (LP_NORMAL << 15) | (((body as u32) & 0x7FFF) << 17); + page[slot..slot + SIZE_OF_ITEM_ID].copy_from_slice(&raw.to_le_bytes()); + written = i + 1; + } + let pd_lower = SIZE_OF_PAGE_HEADER + written * SIZE_OF_ITEM_ID; + let pd_upper = PAGE_BYTES - written * body; + page[12..14].copy_from_slice(&(pd_lower as u16).to_le_bytes()); + page[14..16].copy_from_slice(&(pd_upper as u16).to_le_bytes()); + page +} + +/// One tar part holding `segments_per_part` `base//` entries. +async fn build_part(shape: Shape, part: usize) -> Vec { + use tokio::io::AsyncWriteExt; + + let mut builder = tokio_tar::Builder::new(Vec::new()); + for s in 0..shape.segments_per_part { + let filenode = FIRST_FILENODE + (part * shape.segments_per_part + s) as u32; + let mut body = Vec::with_capacity(shape.pages_per_segment * PAGE_BYTES); + for p in 0..shape.pages_per_segment { + body.extend_from_slice(&heap_page( + shape.tuples_per_page, + shape.columns, + (p * 1_000) as i32, + )); + } + let mut header = tokio_tar::Header::new_gnu(); + header + .set_path(format!("base/{DB_NODE}/{filenode}")) + .unwrap(); + header.set_size(body.len() as u64); + header.set_mode(0o600); + header.set_entry_type(tokio_tar::EntryType::Regular); + header.set_cksum(); + builder + .append(&header, std::io::Cursor::new(body)) + .await + .unwrap(); + } + builder.finish().await.unwrap(); + let mut out = builder.into_inner().await.unwrap(); + out.flush().await.unwrap(); + out +} + +/// Object-store-shaped source over pre-built tars: one `pump_tar_to_sink` +/// per part, `parallelism` in flight, so the shared sink sees the same +/// interleaving a real fan-out gives it. +struct MemSource { + parts: Vec>, + parallelism: usize, +} + +#[async_trait::async_trait] +impl BackupSource for MemSource { + async fn run( + self: Box, + data_dir: std::path::PathBuf, + sink: Arc, + stats: Arc, + ) -> anyhow::Result<(StartInfo, EndInfo)> { + use futures::{StreamExt, TryStreamExt}; + + let start = StartInfo { + start_lsn: START_LSN, + timeline: 1, + tablespaces: Vec::::new(), + }; + let end = EndInfo { + end_lsn: START_LSN + 0x1000, + timeline: 1, + }; + sink.start(&start).await?; + let target = Arc::new(PumpTarget::new(data_dir, sink.clone(), stats)); + let MemSource { parts, parallelism } = *self; + futures::stream::iter(parts) + .map(|blob| { + let target = target.clone(); + async move { + let mut archive = tokio_tar::Archive::new(std::io::Cursor::new(blob)); + pump_tar_to_sink(&mut archive, &target).await + } + }) + .buffer_unordered(parallelism) + .try_collect::>() + .await?; + sink.finish(&end).await?; + Ok((start, end)) + } +} + +struct Report { + label: String, + elapsed_secs: f64, + tuples: u64, + bytes: u64, + detail: Vec<(&'static str, f64)>, +} + +impl Report { + fn print(&self) { + println!( + "{:<10} {:>8.3}s {:>10.1} MiB/s {:>12.0} tuples/s ({} tuples, {:.1} MiB)", + self.label, + self.elapsed_secs, + self.bytes as f64 / self.elapsed_secs / (1 << 20) as f64, + self.tuples as f64 / self.elapsed_secs, + self.tuples, + self.bytes as f64 / (1 << 20) as f64, + ); + for (name, v) in &self.detail { + println!(" {name:<26} {v:>10.3}"); + } + } +} + +async fn bench_pump(shape: Shape) -> Report { + let mut parts = Vec::with_capacity(shape.parts); + for p in 0..shape.parts { + parts.push(build_part(shape, p).await); + } + + let mut catalog = CatalogMap::new(); + for i in 0..shape.segments() { + catalog.insert(descriptor(FIRST_FILENODE + i as u32, shape.columns)); + } + + let tmp = tempfile::tempdir().unwrap(); + let cfg = BootstrapConfig::new(tmp.path().join("data")); + let progress = cfg.progress.clone(); + let source = Box::new(MemSource { + parts, + parallelism: shape.parallelism, + }); + + let started = Instant::now(); + let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog, false); + let drained = tokio::spawn(async move { + let mut n = 0u64; + while let Some(slab) = rx.recv().await { + // Touch the payload so the decode can't be optimised away + n += slab.iter().map(|t| t.columns.len() as u64).sum::(); + } + n + }); + pump.await.unwrap().unwrap(); + let touched = drained.await.unwrap(); + let elapsed = started.elapsed().as_secs_f64(); + + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64; + Report { + label: "pump".into(), + elapsed_secs: elapsed, + tuples: touched / shape.columns as u64, + bytes: ld(&progress.pump.bytes_tapped) as u64, + detail: vec![ + ("pages_walked", ld(&progress.page_walk.pages_walked)), + ("decode_secs", ld(&progress.page_walk.decode_nanos) / 1e9), + ("tap_secs", ld(&progress.pump.sink_chunk_nanos) / 1e9), + ( + "channel_block_secs", + ld(&progress.page_walk.channel_block_nanos) / 1e9, + ), + ], + } +} + +async fn bench_drain(shape: Shape) -> Report { + let rels: Vec> = (0..shape.segments()) + .map(|i| descriptor(FIRST_FILENODE + i as u32, shape.columns)) + .collect(); + let mut catalog = CatalogMap::new(); + let mut tables = std::collections::HashMap::new(); + for r in &rels { + catalog.insert(r.clone()); + tables.insert(r.rel_name.clone(), mapping(r.oid, shape.columns)); + } + let handle = walshadow::mapping::mapping_handle(tables.into_iter().collect()); + + let (msg_tx, ack, tail) = + walshadow::pipeline::tail::spawn_null(Arc::new(Monotone::::new(0))); + let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::>(16); + + let per_rel = shape.pages_per_segment as u64 * shape.tuples_per_page as u64; + let columns = shape.columns; + let feeder = tokio::spawn(async move { + for r in rels { + // Slab at the walk's grain so the drain sees production shape + for slab_start in (0..per_rel).step_by(shape.tuples_per_page * 16) { + let end = (slab_start + (shape.tuples_per_page * 16) as u64).min(per_rel); + let slab: Vec = (slab_start..end) + .map(|i| BackfillTuple { + rfn: r.rfn, + xid: 100 + i as u32, + xmax: 0, + infomask: 0, + source_lsn: START_LSN, + blkno: (i / 100) as u32, + offnum: (i % 100) as u16 + 1, + columns: (0..columns) + .map(|c| Some(ColumnValue::Int4(i as i32 + c as i32))) + .collect(), + }) + .collect(); + if tup_tx.send(slab).await.is_err() { + return; + } + } + } + }); + + let tmp = tempfile::tempdir().unwrap(); + let stats = Arc::new(walshadow::ch_emitter::EmitterStats::default()); + let started = Instant::now(); + let outcome = bootstrap::drain( + tup_rx, + catalog, + handle, + msg_tx.clone(), + ack.clone(), + stats.clone(), + ToastResolver::disabled(), + walshadow::spool::DeferredSpool::new( + tmp.path().join("deferred.bin"), + walshadow::spool::DEFERRED_SPOOL_MEM_MAX, + ), + Default::default(), + None, + std::collections::HashSet::new(), + ) + .await + .unwrap(); + let elapsed = started.elapsed().as_secs_f64(); + feeder.await.unwrap(); + drop(msg_tx); + drop(ack); + tail.join().await; + + Report { + label: "drain".into(), + elapsed_secs: elapsed, + tuples: outcome.rows_routed, + // Int4 payload only; the interesting rate here is rows, not bytes + bytes: outcome.rows_routed * 4 * shape.columns as u64, + detail: vec![("seqs", outcome.next_seq as f64)], + } +} + +fn usage() -> ! { + eprintln!( + "usage: bootstrap_pump [--list] [--stage pump|drain|all] [--parts N] \ + [--segments-per-part N] [--pages N] [--tuples-per-page N] [--columns N] \ + [--parallelism N]" + ); + std::process::exit(2) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + // nextest --all-targets lists benches by running them with --list + if args.iter().any(|a| a == "--list") { + println!("bootstrap_pump: benchmark"); + return; + } + let mut shape = Shape::default(); + let mut stage = "all".to_string(); + let mut i = 0; + while i < args.len() { + let val = |s: &Option<&String>| -> usize { + s.and_then(|v| v.parse().ok()).unwrap_or_else(|| usage()) + }; + let next = args.get(i + 1); + match args[i].as_str() { + "--stage" => stage = next.cloned().unwrap_or_else(|| usage()), + "--parts" => shape.parts = val(&next).max(1), + "--segments-per-part" => shape.segments_per_part = val(&next).max(1), + "--pages" => shape.pages_per_segment = val(&next).max(1), + "--tuples-per-page" => shape.tuples_per_page = val(&next).max(1), + "--columns" => shape.columns = val(&next).max(1), + "--parallelism" => shape.parallelism = val(&next).max(1), + // cargo bench passes --bench through + "--bench" => { + i += 1; + continue; + } + _ => usage(), + } + i += 2; + } + + let rt = tokio::runtime::Runtime::new().unwrap(); + println!( + "shape: {} segments x {} pages x {} tuples x {} int4 cols, {} parts, parallelism {}", + shape.segments(), + shape.pages_per_segment, + shape.tuples_per_page, + shape.columns, + shape.parts, + shape.parallelism, + ); + println!( + " {:.1} MiB of heap, {} tuples", + shape.bytes() as f64 / (1 << 20) as f64, + shape.tuples(), + ); + if stage == "pump" || stage == "all" { + rt.block_on(bench_pump(shape)).print(); + } + if stage == "drain" || stage == "all" { + rt.block_on(bench_drain(shape)).print(); + } +} diff --git a/benches/oracle_batch.rs b/benches/oracle_batch.rs new file mode 100644 index 00000000..03e49188 --- /dev/null +++ b/benches/oracle_batch.rs @@ -0,0 +1,392 @@ +//! Oracle batch cost, by stage. +//! +//! - `frame`: the two payload-sized memory passes a request used to make, no +//! Postgres. One is the request copy into a second buffer the bridge +//! framed; the other is the zeroing pass a fresh response buffer pays +//! before `read_exact` overwrites it. Both scale with the batch, and +//! `ORACLE_BATCH_SEAL_BYTES` puts that at 32 MiB +//! - `roundtrip`: a real shadow Postgres and a real `ENCODE_NATIVE`. Reports +//! what one batch costs end to end, alone and with the bridge pool +//! saturated, so the memory passes above have something to be a fraction +//! of. Also prices a column of `Literal` cells — bytes the daemon already +//! rendered — against building the same ClickHouse column locally +//! +//! ```text +//! cargo bench --bench oracle_batch -- --stage frame --bytes 33554432 +//! cargo bench --bench oracle_batch -- --stage roundtrip --rows 100000 --workers 4 +//! ``` +//! +//! `roundtrip` needs `initdb` on PATH and `pgext/walshadow.so` built against +//! that major; it says so and skips otherwise. +//! +//! `harness = false`, so nextest lists it by answering `--list`. + +// Matches `walshadow-stream`, whose allocator is not glibc's. Request +// buffers here are allocated and freed at 32 MiB, which is exactly where +// allocators differ most +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +use std::path::PathBuf; +use std::process::Command; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clickhouse_c::{Allocator, ColumnBuilder}; +use walshadow::bridge::Bridge; +use walshadow::oracle::{Oracle, OracleCell, OracleColumnBuf, OracleRequestColumn}; +use walshadow::schema::NUMERICOID; +use walshadow::shadow::{BridgeConf, Shadow, ShadowConfig}; + +/// Port outside the integration suites' allocation +const PG_PORT: u16 = 55450; +/// Short-form numeric for 42: header 0x8000, one base-10000 digit +fn numeric_42() -> Vec { + let mut out = 0x8000u16.to_le_bytes().to_vec(); + out.extend_from_slice(&42i16.to_le_bytes()); + out +} + +/// WKT a PostGIS 2-D point renders to, ie a `Literal` cell +const WKT: &[u8] = b"POINT(-73.987654 40.748817)"; + +struct Args { + stage: String, + bytes: usize, + rows: usize, + workers: usize, + iters: usize, +} + +impl Default for Args { + fn default() -> Self { + Self { + stage: "all".into(), + bytes: 32 << 20, + rows: 100_000, + workers: 4, + iters: 8, + } + } +} + +fn usage() -> ! { + eprintln!( + "usage: --stage frame|roundtrip|all [--bytes N] [--rows N] [--workers N] [--iters N]" + ); + std::process::exit(2); +} + +/// Mean over `iters`, minus a warmup pass whose cost is first-touch +fn timed(iters: usize, mut f: impl FnMut()) -> Duration { + f(); + let started = Instant::now(); + for _ in 0..iters { + f(); + } + started.elapsed() / iters as u32 +} + +fn mib_per_s(bytes: usize, d: Duration) -> f64 { + bytes as f64 / (1 << 20) as f64 / d.as_secs_f64() +} + +/// The two payload-sized passes a batch no longer makes +fn bench_frame(args: &Args) { + let n = args.bytes; + let src = vec![0xa5u8; n]; + + // Request side, before: payload into its own buffer, then a second + // buffer of 4 + len the whole payload is copied into so one write_all + // ships it. After: cells go straight past a reserved prefix, so neither + // the allocation nor the copy happens + let copy = timed(args.iters, || { + let mut frame = Vec::with_capacity(5 + n); + frame.extend_from_slice(&[0u8; 5]); + frame.extend_from_slice(&src); + std::hint::black_box(frame.len()); + }); + + // Response side: `vec![0u8; len]` zeroes bytes read_exact then + // overwrites. A recycled buffer read into its spare capacity pays + // neither the allocation nor the zeroing + let fresh = timed(args.iters, || { + let mut v = vec![0u8; n]; + v.copy_from_slice(&src); + std::hint::black_box(v.len()); + }); + let mut scratch: Vec = Vec::with_capacity(n); + let recycled = timed(args.iters, || { + scratch.clear(); + scratch.extend_from_slice(&src); + std::hint::black_box(scratch.len()); + }); + + println!( + "frame stage, {:.0} MiB payload", + n as f64 / (1 << 20) as f64 + ); + println!( + " request second buffer + copy {:>8.3} ms ({:.0} MiB/s)", + copy.as_secs_f64() * 1e3, + mib_per_s(n, copy), + ); + println!( + " response vec![0u8; len] + fill {:>8.3} ms ({:.0} MiB/s)", + fresh.as_secs_f64() * 1e3, + mib_per_s(n, fresh), + ); + println!( + " response recycled + fill {:>8.3} ms ({:.0} MiB/s)", + recycled.as_secs_f64() * 1e3, + mib_per_s(n, recycled), + ); + println!( + " zeroing pass alone {:>8.3} ms", + (fresh.as_secs_f64() - recycled.as_secs_f64()) * 1e3, + ); +} + +fn pg_available() -> bool { + Command::new("initdb") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn pgext_dir() -> Option { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("pgext"); + dir.join("walshadow.so").is_file().then_some(dir) +} + +struct StopOnDrop { + sh: Shadow, +} + +impl Drop for StopOnDrop { + fn drop(&mut self) { + let _ = self.sh.stop(); + } +} + +fn start_shadow(tmp: &tempfile::TempDir, lib_dir: PathBuf, workers: usize) -> StopOnDrop { + let mut cfg = ShadowConfig::new(tmp.path().join("data"), tmp.path().join("filtered")); + cfg.port = PG_PORT; + cfg.socket_dir = tmp.path().join("sock"); + cfg.ctl_timeout = Duration::from_secs(60); + let mut bridge = BridgeConf::in_dir(&cfg.socket_dir); + bridge.library_dir = Some(lib_dir); + bridge.workers = workers; + cfg.bridge = Some(bridge); + std::fs::create_dir_all(&cfg.filter_out_dir).unwrap(); + std::fs::create_dir_all(&cfg.socket_dir).unwrap(); + let sh = Shadow::new(cfg); + sh.initdb().expect("initdb"); + sh.write_base_conf().expect("write_base_conf"); + sh.start().expect("start"); + StopOnDrop { sh } +} + +fn cells(oid: u32, body: &[u8], rows: usize, literal: bool) -> OracleColumnBuf { + let mut buf = OracleColumnBuf::new(oid, -1); + for _ in 0..rows { + buf.push(if literal { + OracleCell::Literal(body.to_vec()) + } else { + OracleCell::DiskRaw(body.to_vec()) + }); + } + buf +} + +async fn one_batch(oracle: &Oracle, buf: &OracleColumnBuf, rows: usize) -> Duration { + let columns = [OracleRequestColumn { + ordinal: 0, + name: "c0", + target_type: "String", + buf, + }]; + let started = Instant::now(); + let block = oracle + .encode_batch(&columns, rows, Allocator::stdlib()) + .await + .expect("oracle answers"); + let elapsed = started.elapsed(); + assert_eq!( + block + .column(0) + .and_then(|c| c.string()) + .expect("string") + .0 + .len(), + rows, + ); + elapsed +} + +/// Mean batch latency, `iters` batches in flight `concurrency` at a time +async fn batches( + oracle: Arc, + buf: Arc, + rows: usize, + iters: usize, + concurrency: usize, +) -> (Duration, Duration) { + let started = Instant::now(); + let mut sum = Duration::ZERO; + let mut left = iters; + while left > 0 { + let wave = concurrency.min(left); + let mut set = Vec::new(); + for _ in 0..wave { + let oracle = oracle.clone(); + let buf = buf.clone(); + set.push(tokio::spawn( + async move { one_batch(&oracle, &buf, rows).await }, + )); + } + for h in set { + sum += h.await.unwrap(); + } + left -= wave; + } + (sum / iters as u32, started.elapsed() / iters as u32) +} + +/// `ColumnBuilder::string` over the same cells, ie what a pure-`Literal` +/// column would cost if it never left the daemon +fn local_string_column(rows: usize, body: &[u8]) -> Duration { + let iters = 8; + timed(iters, || { + let mut offsets: Vec = Vec::with_capacity(rows); + let mut data: Vec = Vec::with_capacity(rows * body.len()); + for _ in 0..rows { + data.extend_from_slice(body); + offsets.push(data.len() as u64); + } + let col = ColumnBuilder::string(&offsets, &data, rows).expect("string column"); + std::hint::black_box(col.n_rows()); + }) +} + +async fn bench_roundtrip(args: &Args) { + let Some(lib_dir) = pgext_dir() else { + println!("roundtrip stage: skip, pgext/walshadow.so missing (make -C pgext)"); + return; + }; + if !pg_available() { + println!("roundtrip stage: skip, no initdb on PATH"); + return; + } + let tmp = tempfile::tempdir().unwrap(); + let guard = start_shadow(&tmp, lib_dir, args.workers); + let socket = guard.sh.bridge_socket().expect("bridge configured"); + + let disk = Arc::new(cells(NUMERICOID, &numeric_42(), args.rows, false)); + let literal = Arc::new(cells(NUMERICOID, WKT, args.rows, true)); + let req_bytes = |b: &OracleColumnBuf| b.approx_size(); + + for workers in [1, args.workers] { + let bridge = Arc::new( + walshadow::bridge::connect_with_budget(socket, workers, Duration::from_secs(30)) + .await + .expect("bridge connect"), + ); + let oracle = Arc::new(Oracle::new(bridge.clone())); + let serial = batches(oracle.clone(), disk.clone(), args.rows, args.iters, 1).await; + let parallel = batches( + oracle.clone(), + disk.clone(), + args.rows, + args.iters, + workers.max(1), + ) + .await; + println!( + "roundtrip stage, {} DiskRaw numeric cells ({:.1} MiB request), {} sockets", + args.rows, + req_bytes(&disk) as f64 / (1 << 20) as f64, + bridge.pool_size(), + ); + println!( + " one batch at a time {:>8.2} ms/batch", + serial.1.as_secs_f64() * 1e3, + ); + println!( + " {:>2} in flight {:>8.2} ms/batch wall, {:.2} ms latency", + workers, + parallel.1.as_secs_f64() * 1e3, + parallel.0.as_secs_f64() * 1e3, + ); + } + + // Cells the daemon already rendered, which the resolver now builds + // locally: this is what shipping them used to cost + let bridge = Arc::new( + walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) + .await + .expect("bridge connect"), + ); + let oracle = Arc::new(Oracle::new(bridge)); + let shipped = batches(oracle, literal.clone(), args.rows, args.iters, 1).await; + let built = local_string_column(args.rows, WKT); + println!( + "literal column, {} cells ({:.1} MiB)", + args.rows, + req_bytes(&literal) as f64 / (1 << 20) as f64, + ); + println!( + " through the oracle {:>8.2} ms/batch", + shipped.1.as_secs_f64() * 1e3, + ); + println!( + " built locally {:>8.2} ms/batch", + built.as_secs_f64() * 1e3, + ); +} + +fn main() { + let argv: Vec = std::env::args().skip(1).collect(); + // nextest --all-targets lists benches by running them with --list + if argv.iter().any(|a| a == "--list") { + println!("oracle_batch: benchmark"); + return; + } + let mut args = Args::default(); + let mut i = 0; + while i < argv.len() { + let val = |s: &Option<&String>| -> usize { + s.and_then(|v| v.parse().ok()).unwrap_or_else(|| usage()) + }; + let next = argv.get(i + 1); + match argv[i].as_str() { + "--stage" => args.stage = next.cloned().unwrap_or_else(|| usage()), + "--bytes" => args.bytes = val(&next).max(1), + "--rows" => args.rows = val(&next).max(1), + "--workers" => args.workers = val(&next).clamp(1, 8), + "--iters" => args.iters = val(&next).max(1), + // cargo bench passes --bench through + "--bench" => { + i += 1; + continue; + } + _ => usage(), + } + i += 2; + } + + if args.stage == "frame" || args.stage == "all" { + bench_frame(&args); + } + if args.stage == "roundtrip" || args.stage == "all" { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(bench_roundtrip(&args)); + } +} + +/// Silences the unused-import lint when only `frame` is compiled in +#[allow(dead_code)] +fn _bridge_type_is_used(_: &Bridge) {} diff --git a/pgext/worker.c b/pgext/worker.c index 4f9fd40e..a52be27a 100644 --- a/pgext/worker.c +++ b/pgext/worker.c @@ -43,9 +43,22 @@ PG_MODULE_MAGIC; #define WS_MAX_CONNS 8 +/* + * Each worker serves one request at a time, so oracle throughput is one + * backend's conversion rate. `walshadow.bridge_workers` registers copies; + * worker 0 keeps the bare `socket_path` so a single-worker deployment and + * every catalog read are untouched, worker i listens on `socket_path.i`. + */ +#define WS_MAX_WORKERS 8 #define WS_LISTEN_BACKLOG 16 #define WS_IDLE_POLL_MS 1000 #define WS_MAX_SCAN_OIDS 65536 +/* + * A 32 MiB request against default socket buffers costs hundreds of + * EAGAIN round trips; ask for a wide window and accept whatever the + * kernel grants (it halves the request and clamps to wmem_max). + */ +#define WS_SOCKBUF_BYTES (4 * 1024 * 1024) /* wait-set positions, in the order ws_build_wait_set adds them */ #define WS_POS_LATCH 0 @@ -53,10 +66,27 @@ PG_MODULE_MAGIC; #define WS_POS_LISTEN 2 #define WS_POS_CONN0 3 +/* ... and in the per-connection io set */ +#define WS_IO_POS_LATCH 0 +#define WS_IO_POS_PM_DEATH 1 +#define WS_IO_POS_SOCKET 2 + +/* + * One accepted connection. `io` is created once and its socket event mask + * flipped between readable and writeable, because building a fresh epoll + * set per EAGAIN is what a large transfer would otherwise pay for. + */ +typedef struct WsConn +{ + pgsocket fd; + WaitEventSet *io; +} WsConn; + PGDLLEXPORT void ws_worker_main(Datum main_arg); static char *ws_socket_path = NULL; static char *ws_database = NULL; +static int ws_bridge_workers = 1; static int ws_io_timeout_ms = 30000; static int ws_lock_timeout_ms = 1000; @@ -175,20 +205,34 @@ ws_listen(const char *path) return fd; } +static WaitEventSet * +ws_create_wait_set(int nevents) +{ +#if PG_VERSION_NUM >= 170000 + return CreateWaitEventSet(NULL, nevents); +#else + return CreateWaitEventSet(TopMemoryContext, nevents); +#endif +} + /* - * Wait for `event` on one socket. `false` means the caller should abandon the - * connection: shutdown was requested. + * Wait for `event` on one connection, reusing its own wait set. `false` + * means the caller should abandon the connection: shutdown was requested. */ static bool -ws_wait_socket(pgsocket fd, int event, long timeout_ms) +ws_wait_conn(WsConn *conn, int event, long timeout_ms) { - int rc; + WaitEvent events[3]; + int nready; + int i; - rc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT | event, - fd, timeout_ms, PG_WAIT_EXTENSION); - if (rc & WL_LATCH_SET) + ModifyWaitEvent(conn->io, WS_IO_POS_SOCKET, event, NULL); + nready = WaitEventSetWait(conn->io, timeout_ms, events, + lengthof(events), PG_WAIT_EXTENSION); + for (i = 0; i < nready; i++) { + if (events[i].pos != WS_IO_POS_LATCH) + continue; ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); if (ConfigReloadPending) @@ -203,10 +247,11 @@ ws_wait_socket(pgsocket fd, int event, long timeout_ms) } static bool -ws_read_exact(pgsocket fd, char *buf, size_t len) +ws_read_exact(WsConn *conn, char *buf, size_t len) { size_t got = 0; TimestampTz deadline; + pgsocket fd = conn->fd; deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), ws_io_timeout_ms); @@ -242,17 +287,18 @@ ws_read_exact(pgsocket fd, char *buf, size_t len) ws_io_timeout_ms))); return false; } - if (!ws_wait_socket(fd, WL_SOCKET_READABLE, wait_ms)) + if (!ws_wait_conn(conn, WL_SOCKET_READABLE, wait_ms)) return false; } return true; } static bool -ws_write_all(pgsocket fd, const char *buf, size_t len) +ws_write_all(WsConn *conn, const char *buf, size_t len) { size_t sent = 0; TimestampTz deadline; + pgsocket fd = conn->fd; deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), ws_io_timeout_ms); @@ -288,7 +334,7 @@ ws_write_all(pgsocket fd, const char *buf, size_t len) ws_io_timeout_ms))); return false; } - if (!ws_wait_socket(fd, WL_SOCKET_WRITEABLE, wait_ms)) + if (!ws_wait_conn(conn, WL_SOCKET_WRITEABLE, wait_ms)) return false; } return true; @@ -448,7 +494,7 @@ ws_dispatch(StringInfo req, StringInfo resp) * `false` closes the connection. */ static bool -ws_serve_request(pgsocket fd) +ws_serve_request(WsConn *conn) { uint32 hdr; uint32 len; @@ -457,7 +503,7 @@ ws_serve_request(pgsocket fd) MemoryContext oldctx; bool ok = false; - if (!ws_read_exact(fd, (char *) &hdr, sizeof(hdr))) + if (!ws_read_exact(conn, (char *) &hdr, sizeof(hdr))) return false; len = pg_ntoh32(hdr); @@ -476,7 +522,7 @@ ws_serve_request(pgsocket fd) initStringInfo(&req); enlargeStringInfo(&req, (int) len); - if (ws_read_exact(fd, req.data, len)) + if (ws_read_exact(conn, req.data, len)) { req.len = (int) len; req.data[len] = '\0'; @@ -487,8 +533,8 @@ ws_serve_request(pgsocket fd) ws_dispatch(&req, &resp); hdr = pg_hton32((uint32) resp.len); - ok = ws_write_all(fd, (char *) &hdr, sizeof(hdr)) && - ws_write_all(fd, resp.data, (size_t) resp.len); + ok = ws_write_all(conn, (char *) &hdr, sizeof(hdr)) && + ws_write_all(conn, resp.data, (size_t) resp.len); } MemoryContextSwitchTo(oldctx); @@ -501,32 +547,54 @@ ws_serve_request(pgsocket fd) * ------------------------------------------------------------------------- */ /* - * Rebuilt per iteration: the wait-set API has no portable event removal, and - * this loop is idle-dominated. + * The wait-set API has no portable event removal, so the set is rebuilt on + * a membership change and reused across every iteration in between. Under + * load that is one epoll set per connect/disconnect rather than per request. */ static WaitEventSet * -ws_build_wait_set(pgsocket listen_fd, const pgsocket *conns, int nconns) +ws_build_wait_set(pgsocket listen_fd, const WsConn *conns, int nconns) { - WaitEventSet *set; + WaitEventSet *set = ws_create_wait_set(nconns + WS_POS_CONN0); int i; -#if PG_VERSION_NUM >= 170000 - set = CreateWaitEventSet(NULL, nconns + WS_POS_CONN0); -#else - set = CreateWaitEventSet(CurrentMemoryContext, nconns + WS_POS_CONN0); -#endif AddWaitEventToSet(set, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); AddWaitEventToSet(set, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); AddWaitEventToSet(set, WL_SOCKET_READABLE, listen_fd, NULL, NULL); for (i = 0; i < nconns; i++) - AddWaitEventToSet(set, WL_SOCKET_READABLE, conns[i], NULL, NULL); + AddWaitEventToSet(set, WL_SOCKET_READABLE, conns[i].fd, NULL, NULL); return set; } +/* + * Widen the socket buffers so a multi-megabyte frame crosses in a handful + * of syscalls. Advisory: a kernel that refuses leaves the default, which + * only costs more EAGAIN waits. + */ static void -ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) +ws_widen_sockbufs(pgsocket fd) +{ + int want = WS_SOCKBUF_BYTES; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (char *) &want, sizeof(want)) < 0 || + setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (char *) &want, sizeof(want)) < 0) + ereport(DEBUG1, + (errcode_for_socket_access(), + errmsg("walshadow: could not widen socket buffers: %m"))); +} + +/* + * Membership generation, bumped on every accept and drop. The serve loop + * keys its cached wait set off this, not off nconns: an accept and a drop + * in the same iteration leave nconns unchanged over a different fd set, + * and a stale set would then poll a closed fd and miss the new one. + */ +static uint64 ws_conn_gen = 0; + +static bool +ws_accept(pgsocket listen_fd, WsConn *conns, int *nconns) { pgsocket fd; + WsConn *conn; fd = accept(listen_fd, NULL, NULL); if (fd == PGINVALID_SOCKET) @@ -535,7 +603,7 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) ereport(LOG, (errcode_for_socket_access(), errmsg("walshadow: accept failed: %m"))); - return; + return false; } if (*nconns >= WS_MAX_CONNS) { @@ -543,7 +611,7 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) (errmsg("walshadow: refusing connection, %d already open", WS_MAX_CONNS))); closesocket(fd); - return; + return false; } if (!pg_set_noblock(fd)) { @@ -551,27 +619,39 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) (errcode_for_socket_access(), errmsg("walshadow: could not set client socket non-blocking: %m"))); closesocket(fd); - return; + return false; } - conns[(*nconns)++] = fd; + ws_widen_sockbufs(fd); + + conn = &conns[(*nconns)++]; + ws_conn_gen++; + conn->fd = fd; + conn->io = ws_create_wait_set(3); + AddWaitEventToSet(conn->io, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); + AddWaitEventToSet(conn->io, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); + AddWaitEventToSet(conn->io, WL_SOCKET_READABLE, fd, NULL, NULL); + return true; } static void -ws_drop_conn(pgsocket *conns, int *nconns, int idx) +ws_drop_conn(WsConn *conns, int *nconns, int idx) { - closesocket(conns[idx]); + FreeWaitEventSet(conns[idx].io); + closesocket(conns[idx].fd); conns[idx] = conns[--*nconns]; + ws_conn_gen++; } static void ws_serve_loop(pgsocket listen_fd) { - pgsocket conns[WS_MAX_CONNS]; + WsConn conns[WS_MAX_CONNS]; int nconns = 0; + WaitEventSet *set = NULL; + uint64 set_gen = 0; for (;;) { - WaitEventSet *set; WaitEvent events[WS_MAX_CONNS + WS_POS_CONN0]; int nready; int i; @@ -586,10 +666,15 @@ ws_serve_loop(pgsocket listen_fd) if (ShutdownRequestPending) break; - set = ws_build_wait_set(listen_fd, conns, nconns); + if (set == NULL || set_gen != ws_conn_gen) + { + if (set != NULL) + FreeWaitEventSet(set); + set = ws_build_wait_set(listen_fd, conns, nconns); + set_gen = ws_conn_gen; + } nready = WaitEventSetWait(set, WS_IDLE_POLL_MS, events, lengthof(events), PG_WAIT_EXTENSION); - FreeWaitEventSet(set); for (i = 0; i < nready; i++) { @@ -611,12 +696,14 @@ ws_serve_loop(pgsocket listen_fd) if (serve >= 0 && serve < nconns) { pgstat_report_activity(STATE_RUNNING, "walshadow request"); - if (!ws_serve_request(conns[serve])) + if (!ws_serve_request(&conns[serve])) ws_drop_conn(conns, &nconns, serve); pgstat_report_activity(STATE_IDLE, NULL); } } + if (set != NULL) + FreeWaitEventSet(set); while (nconns > 0) ws_drop_conn(conns, &nconns, 0); closesocket(listen_fd); @@ -625,8 +712,10 @@ ws_serve_loop(pgsocket listen_fd) void ws_worker_main(Datum main_arg) { + int idx = DatumGetInt32(main_arg); pgsocket listen_fd; char buf[32]; + char path[MAXPGPATH]; pqsignal(SIGTERM, SignalHandlerForShutdownRequest); pqsignal(SIGHUP, SignalHandlerForConfigReload); @@ -659,10 +748,14 @@ ws_worker_main(Datum main_arg) "walshadow request", ALLOCSET_DEFAULT_SIZES); - listen_fd = ws_listen(ws_socket_path); + if (idx == 0) + strlcpy(path, ws_socket_path, sizeof(path)); + else + snprintf(path, sizeof(path), "%s.%d", ws_socket_path, idx); + listen_fd = ws_listen(path); ereport(LOG, (errmsg("walshadow bridge listening on \"%s\" (proto %d)", - ws_socket_path, WS_PROTO_VERSION))); + path, WS_PROTO_VERSION))); ws_serve_loop(listen_fd); @@ -680,6 +773,7 @@ void _PG_init(void) { BackgroundWorker worker; + int i; /* * Worker registration and its postmaster-scoped GUCs are only legal @@ -716,20 +810,33 @@ _PG_init(void) 1000, 0, INT_MAX, PGC_POSTMASTER, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable("walshadow.bridge_workers", + "Bridge workers to register.", + "Worker 0 listens on socket_path, worker i on " + "socket_path.i. Each serves one request at a " + "time, so this bounds concurrent decode.", + &ws_bridge_workers, + 1, 1, WS_MAX_WORKERS, + PGC_POSTMASTER, 0, + NULL, NULL, NULL); MarkGUCPrefixReserved("walshadow"); if (ws_socket_path == NULL || ws_socket_path[0] == '\0') return; - memset(&worker, 0, sizeof(worker)); - worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - /* Catalog reads need a database connection, so not before consistency */ - worker.bgw_start_time = BgWorkerStart_ConsistentState; - worker.bgw_restart_time = 5; - strlcpy(worker.bgw_library_name, "walshadow", BGW_MAXLEN); - strlcpy(worker.bgw_function_name, "ws_worker_main", BGW_MAXLEN); - strlcpy(worker.bgw_name, "walshadow bridge", BGW_MAXLEN); - strlcpy(worker.bgw_type, "walshadow bridge", BGW_MAXLEN); - RegisterBackgroundWorker(&worker); + for (i = 0; i < ws_bridge_workers; i++) + { + memset(&worker, 0, sizeof(worker)); + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; + /* Catalog reads need a database connection, so not before consistency */ + worker.bgw_start_time = BgWorkerStart_ConsistentState; + worker.bgw_restart_time = 5; + worker.bgw_main_arg = Int32GetDatum(i); + strlcpy(worker.bgw_library_name, "walshadow", BGW_MAXLEN); + strlcpy(worker.bgw_function_name, "ws_worker_main", BGW_MAXLEN); + snprintf(worker.bgw_name, BGW_MAXLEN, "walshadow bridge %d", i); + strlcpy(worker.bgw_type, "walshadow bridge", BGW_MAXLEN); + RegisterBackgroundWorker(&worker); + } } diff --git a/plans/GLOSSARY.md b/plans/GLOSSARY.md index 478b4cb2..9f70d23f 100644 --- a/plans/GLOSSARY.md +++ b/plans/GLOSSARY.md @@ -208,9 +208,10 @@ so does the whole segment holding it, since a fork copies the ancestor prefix into a descendant-named file ([failover.md](failover.md)) **FileAction (Keep / Skip / Tap)** — per-backup-file sink decision: land -body under data dir, drain unread, or stream body through `chunk()` -with nothing landing (page walk, pg_xact accumulation) -([bootstrap.md](bootstrap.md)) +body under data dir, drain unread, or stream body into an owned +`EntrySink` with nothing landing (page walk, pg_xact accumulation). +Owned rather than a shared-state lookup so concurrent tar parts share no +lock on the body path ([bootstrap.md](bootstrap.md)) **filter** — per-record keep/drop engine on WalStream: kept records pass (catalog-only blocks retained, CRC32C recomputed), drops NOOP-rewrite in @@ -349,8 +350,8 @@ whole overlay subsystem; `config_table.replicate` opts one table into replication, triggering backfill per `initial_load` ([table selection guide](../docs/table-selection.md), [add_table.md](add_table.md)) -**oracle** — resolver for types walshadow does not decode: shadow PG -converts on-disk bytes into a ClickHouse Native column through the bridge +**oracle** — converter for types walshadow does not decode: shadow PG +turns on-disk bytes into a ClickHouse Native column through the bridge worker's `ENCODE_NATIVE` op, one request per sealed batch. No fallback, a cell it cannot convert fails the batch ([oracle.md](oracle.md)) @@ -400,8 +401,8 @@ backup files, patched with commit/abort records harvested from gap-WAL pre-scan; backs the visibility gate ([add_table.md](add_table.md)) **pipeline** — parallel decode+insert tail: reorder → decode ×M → -batcher → inserter ×N → ack watermark, in `src/pipeline/`; stands up -only with `--ch-config` ([emitter.md](emitter.md)) +batcher → resolve ×W → inserter ×N → ack watermark, in `src/pipeline/`; +stands up only with `--ch-config` ([emitter.md](emitter.md)) **placed** — decoder's per-seq report that all a xact's rows are routed to batcher; seq is done at `placed == acked` @@ -461,6 +462,12 @@ keyless `Default` ([shadow.md](shadow.md)) `RM_XACT_ID` records; assigns seqs, dispatches decode jobs, owns barriers ([emitter.md](emitter.md)) +**resolver pool** — ×W tasks between batcher and inserters answering +sealed batches' oracle columns, W = the bridge's pool width; overlaps a +bridge round trip with other inserters' queries, and builds columns of +already-rendered cells locally instead of shipping them +([emitter.md](emitter.md), [oracle.md](oracle.md)) + **restore_command fallback** — shadow's archive channel (`cp /%f %p`) at segment cadence when walsender wire drops or a slow connection is cut off ([shadow.md](shadow.md)) @@ -570,9 +577,9 @@ or `match` pattern) overrides per relation ([emitter.md](emitter.md), [destination tables guide](../docs/destination-tables.md)) -**tail** — reusable batcher + inserter pool + ack collector unit; WAL -pipeline and bootstrap drain feed the identical tail, `tail.finish` -seals partials and waits all seqs durable +**tail** — reusable batcher + resolver pool + inserter pool + ack +collector unit; WAL pipeline and bootstrap drain feed the identical tail, +`tail.finish` seals partials and waits all seqs durable ([emitter.md](emitter.md), [bootstrap.md](bootstrap.md)) **Tier 1 / 2 / 3** — decoder type matrix: fixed-width (`type_len > 0`) diff --git a/plans/bootstrap.md b/plans/bootstrap.md index 594d4b4a..18e1a51b 100644 --- a/plans/bootstrap.md +++ b/plans/bootstrap.md @@ -43,9 +43,10 @@ for rendered diagram. Five clusters top→bottom: - user heap → `PageWalkSink` (Tap) → decoded 8 KiB at a time - denylist contents → Skip; denylist dir entries themselves → Keep as empty dirs -3. **Drain → CH** (concurrent with step 2) — `PageWalkSink` ships - `BackfillTuple`s through bounded mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP - = 256`, backpressures the tar pump) to +3. **Drain → CH** (concurrent with step 2) — each `PageWalkEntry` ships + slabs of `BackfillTuple`s through a bounded mpsc + (`BOOTSTRAP_TUPLE_CHANNEL_CAP` slabs of `SLAB_BYTES`, backpressures + the tar pump) to `pipeline::bootstrap::drain`, which synthesizes rows `{ op=Insert, commit_lsn=start_lsn }` against the snapshot `CatalogMap` and routes into the shared insert tail (batcher + @@ -160,11 +161,16 @@ pub trait BackupSource: Send { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)>; } ``` +`stats` is the pump's stage attribution (`bytes_tapped`, +`sink_chunk_nanos`), shared with the orchestrator so `/metrics` reads it +while the pump runs. + Public types: - `StartInfo { start_lsn, timeline, tablespaces }` — mirrors @@ -179,9 +185,13 @@ Public types: - `FileMeta { path, size, mode, kind }` — `path` cluster-relative, sanitized against `..` / absolute-root at source-impl boundary (`tar_entry_meta` returns `Ok(None)` on parent-dir traversal) -- `FileAction::{Keep, Skip, Tap}` — sink decision per `begin()`. Keep: - source writes body under `data_dir`; Skip: drain body unread; Tap: - stream body bytes through `chunk()` callbacks, nothing lands +- `FileAction::{Keep, Skip, Tap(Box)}` — sink decision per + `begin()`. Keep: source writes body under `data_dir`; Skip: drain body + unread; Tap: source streams body bytes into the returned owned entry + sink, nothing lands +- `EntrySink` — one tap entry's body consumer, `chunk` then `end`. Owned + by the source task driving that entry, so concurrent entries share no + lock on the body path Per-source guarantees in `src/backup_source.rs` module docs: @@ -194,14 +204,20 @@ Per-source guarantees in `src/backup_source.rs` module docs: 4. `finish()` fires after the last `end()`, carries `end_lsn` 5. Paths are cluster-relative & traversal-safe -Sink trait surface (`BackupSink`): `#[async_trait]` `start` / `begin` -/ `chunk` / `end` / `finish`, `Send` so ObjectStore worker pool can -share `Arc>`. Async surface is load-bearing: -`chunk` fires inside tokio runtime context source drives, and -PageWalkSink's bounded `mpsc::Sender::send(...).await` there is what -backpressures the tar pump against drain throughput (async surface + -bounded channel exist precisely to get this bound; a sync trait + -unbounded channel would not) +Sink trait surface (`BackupSink`): `#[async_trait]` `start` / `begin` / +`finish`, all `&self`, `Send + Sync` so the ObjectStore worker pool +shares one `Arc` with no lock. Routing reads immutable +state and bumps atomics; the body path lives on the per-entry +`EntrySink` the `Tap` decision hands back. A sink that instead held a +shared lock across a body read would serialize every other entry behind +it, which is why the decision is owned rather than a lookup. + +Async surface is load-bearing: `EntrySink::chunk` fires inside the tokio +runtime context the source drives, and `PageWalkEntry`'s bounded +`mpsc::Sender::send(...).await` there is what backpressures the tar pump +against drain throughput (async surface + bounded channel exist +precisely to get this bound; a sync trait + unbounded channel would +not) ## Two source impls @@ -245,11 +261,20 @@ primitives against `DynStorage` bucket (wal-g-compatible layout): (default `min(4, num_cpus)`, overridden by `--bootstrap-object-store-parallelism` / `[bootstrap] object_store_parallelism` when either is set) via `buffer_unordered`, - sharing `Arc>` + sharing one `Arc`. Parallelism reaches the page walk + because each part's tap entries own their own walk state + (`benches/bootstrap_pump.rs` measures it) - `pg_control` parts run as hard barrier after every data part drains — `for key in &control_parts` single-task loop. Multiple control parts is unusual (wal-g emits exactly one) but loop handles it +Part bodies spool to scratch before decode (`spool_backup_part`) so the GET +completes at network speed. Per-entry tap sinks removed the shared-lock +stall the spool was also covering, but it stays: the pipeline downstream +backpressures on ClickHouse by design, and a live body held for however +long CH is the limiter exceeds walrus's 60 s request cap. Costs +`parallelism × part_size` of scratch, compressed + V1 constraint: delta chains error out. Incremented files need disk-resident base to overlay onto via wal-rus's `apply_increment_in_place`, but `Tap` entries never land on disk to be @@ -328,18 +353,37 @@ in which case body drops unread — `PageWalkSink::begin` does this for `pg_control` etc that arrive at user-heap-looking paths or for files whose path does not parse as `base//` -Stats recovery: orchestrator holds two `Arc` clones to same -`Mutex>` — one typed for stats teardown & -one erased (`Arc>`) for source call. `Mutex::into_inner` does not exist (unsized inner); `Arc::try_unwrap` -on typed clone after source returns recovers both inner sinks for -stats reporting +Stats are atomics on `Arc` handles the orchestrator keeps beside the +sink (`BootstrapProgress { pump, page_walk }`, `DiskLanderStats`), so the +sink never has to come back out of the source and `/metrics` reads the +counters while the pump runs. Dropping the last sink `Arc` closes the +tuple-channel sender, which is how a concurrent drain sees EOF ## PageWalkSink -`src/backup_page_walk.rs`. 2A initial-load: Tap user-heap file bodies, -accumulate 8 KiB at a time, walk each full page's `ItemIdData` slots, -decode live tuples through same heap decoder WAL hot path uses +`src/backup_page_walk.rs`. 2A initial-load: `PageWalkSink::begin` +resolves the segment's descriptor and hands back a `PageWalkEntry` that +owns the walk. The entry accumulates body bytes into a `SLAB_BYTES` +slab, walks every complete page in place out of it, carries only a +sub-page remainder into the next slab, and decodes live tuples through +the same heap decoder the WAL hot path uses. + +Two slabs ping-pong per entry: the walk runs on `spawn_blocking` (pure +CPU over a byte slice, no async in it) with at most one in flight, so +the tar reader refills one slab while the other decodes and the async +runtime keeps its workers for the drain stage. Walking in place is what +removes the per-page 8 KiB allocation plus buffer memmove a +`drain(..PAGE_BYTES)` per page costs. + +Per-page counters accumulate in a plain `PageWalkTally` and publish into +the atomic `PageWalkStats` once per slab: a `fetch_add` per tuple, on a +line the reader task also touches, costs more than the decode it counts. + +`begin` also applies the mapped-filenode filter when the orchestrator +passes one (`tap_filenodes`): a relation no mapping routes returns +`Skip`, so its bytes drain off the wire and its pages never decode. +`skip_initial` in the drain stays the authority; the set is a superset +filter, not a second source of truth `heap_decoder::decode_block_data` is exposed as `pub(crate)` for this consumer. On-disk tuple shape carries full `HeapTupleHeaderData` (23 @@ -361,10 +405,12 @@ decoder, exercised from two callers failures bump `tuples_skipped_truncated` so a single torn page does not abort whole bootstrap -`BackfillTuple { rfn, xid, source_lsn, columns }` ships over bounded -mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP`) to orchestrator's drain task. -`source_lsn` is `StartInfo::start_lsn` for every emitted row — every -backfill row tags identically +`BackfillTuple { rfn, xid, source_lsn, columns }` ships in slabs over a +bounded mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP` slabs) to the +orchestrator's drain task. One channel hop per walk slab, not per +tuple; `SLAB_BYTES × CAP` bounds the heap payload in flight per +concurrent segment. `source_lsn` is `StartInfo::start_lsn` for every +emitted row — every backfill row tags identically V1 limits: @@ -391,6 +437,12 @@ V1 limits: - **No 2C CH-side COPY load.** PageWalkSink (2A) is the sole initial-load path; see [Why not 2C](#why-not-2c-ch-side-copy-load) below +`pipeline::bootstrap::drain` coalesces routed rows into +`BatcherMsg::Rows` on the same row/byte dual trigger the streaming +decode pool uses, so a walk slab costs one batcher hop rather than one +per row. The open seq's buffer flushes at an rfn flip before +`placed(prev_seq, rows)` publishes its expected count. + Rfn contiguity buys seq economy, not correctness: `PageWalkSink` emits all rows for one rfn contiguously before moving on, so `pipeline::bootstrap::drain` can synthesize one ack-collector seq per @@ -439,7 +491,7 @@ is the only shape with bounded memory at scale - `seed_in_snapshot(client) -> CatalogMap` — REPEATABLE READ wrapper around `seed_catalog_from_source`. Always COMMITs (read-only xact; commit-vs-rollback is purely about releasing snapshot) -- `spawn_greenfield_bootstrap(cfg, source, catalog_map) -> (mpsc::Receiver, JoinHandle>)` — +- `spawn_greenfield_bootstrap(cfg, source, catalog_map) -> (mpsc::Receiver>, JoinHandle>)` — streaming primitive. Caller drains concurrently with source pump; bounded channel backpressures pump against drain rate, so memory is bounded by `BOOTSTRAP_TUPLE_CHANNEL_CAP`, not source tuple count. @@ -561,6 +613,27 @@ Synchronous `pg_ctl` and `psql` commands run inside wait. `--shadow-socket-dir` and `--shadow-port` configure shadow listener used later by `ShadowCatalog` +## Stage attribution + +`BootstrapProgress` carries the pump's and walk's atomic counters; a ticker +publishes them into `/metrics` while the pump runs and the status loop keeps +rendering their final values afterwards, so a slow initial load stays +attributable after the fact. One `bootstrap stage timings` log line at the +end carries the same numbers: + +- `walshadow_bootstrap_bytes_tapped_total`, + `walshadow_bootstrap_pages_walked_total`, + `walshadow_bootstrap_tuples_emitted_total`, + `walshadow_bootstrap_files_walked_total` — what moved +- `walshadow_bootstrap_files_skipped_unmapped_total` — segments declined at + `begin` because no mapping routes them +- `walshadow_bootstrap_decode_seconds_total` — CPU in the page walk +- `walshadow_bootstrap_channel_block_seconds_total` — the walk waiting on a + free tuple-channel slot, ie emitter drain time seen by the walk +- `walshadow_bootstrap_tap_seconds_total` — time inside the tap entry: + page framing, decode, channel send. Against `decode_seconds` this is + the tap's own overhead + ## Cross-links - [shadow.md](shadow.md) — handoff target. Shadow lifecycle, standby diff --git a/plans/emitter.md b/plans/emitter.md index 5b5df1d3..4f764cdf 100644 --- a/plans/emitter.md +++ b/plans/emitter.md @@ -4,12 +4,12 @@ CH-side ingest is a parallel decode+insert pipeline (`src/pipeline/`): ```text pump -> QueueingRecordSink -> reorder (plan -> execute) -> [decode x M] - -> InsertBatcher -> [inserter x N] -> ClickHouse - \-> ack collector -> emitter_ack_lsn + -> InsertBatcher -> [resolve x W] -> [inserter x N] -> ClickHouse + \-> ack collector -> emitter_ack_lsn ``` Pipeline stages live in `src/pipeline/{reorder,planner,plan_spool, -decode,batcher,inserter,ack,tail,mod}.rs`; encoding primitives +decode,batcher,resolver,inserter,ack,tail,mod}.rs`; encoding primitives (`EmitterConfig`, `TableEncoder`, `TablePlan`, `ColumnBuf`, value encode, `EmitterStats`) in `src/ch_emitter.rs`; DDL in `src/ch_ddl.rs`; PG → CH type mapping in @@ -187,6 +187,21 @@ the rows' admission/value permits, dropped post-insert-ack): 4-table xacts coalesce into one MergeTree part per window instead of one per xact +### Resolver pool — `pipeline/resolver.rs`, ×W + +W = the bridge's pool width, since that is how many requests the shadow +answers at once. Pops sealed `InsertBatch`es, answers their oracle +columns, and pushes `(batch, local, block)` onto a queue one deep per +inserter. Overlaps `T_oracle` with `T_ch` — sequential stages cost +`T_oracle + T_ch / N`, these cost `max(T_oracle / W, T_ch / N)`. Costs +one extra resident batch per inserter + +A batch with no oracle column crosses without touching the bridge, and +so does one whose oracle columns the daemon can answer itself: cells +already rendered locally (PostGIS WKT via `render_ext_columns`) against +a `String` target are the bytes ClickHouse wants, and PG would hand them +straight back. `oracle_local_columns` counts those + ### Inserter pool — `pipeline/inserter.rs`, ×N N `AsyncClient` connections. CH Cloud INSERT cost is mostly RTT + @@ -199,9 +214,9 @@ owned slabs (`TypeAst` cache keyed on `(table, schema_epoch)`; runs one `send_query` + `send_data` + `send_data_end` + drain-to-`EndOfStream` -A batch holding oracle columns resolves them first: one bridge request -before the query opens, spliced into the same block -([oracle.md](oracle.md)) +Oracle columns arrive already answered: the resolver stage hands over a +block to splice ([oracle.md](oracle.md)), so an inserter's ClickHouse +connection is never idle for a bridge round trip Durability invariant: `ack.acked(per_seq)` fires **only after** the drain returns. Until then a connection drop replays the still-owned diff --git a/plans/oracle.md b/plans/oracle.md index af637b7d..e30af10d 100644 --- a/plans/oracle.md +++ b/plans/oracle.md @@ -20,8 +20,9 @@ interprets array elements, hstore pairs, JSON text, null maps, or nested offsets Resolution runs at sealed-batch granularity, one request per `InsertBatch`, -after the batcher has fixed row order and before the inserter opens its -ClickHouse query. It is not best-effort: one Datum the worker cannot convert +in the resolver stage between batcher and inserter pool +([emitter.md](emitter.md)) so a round trip overlaps other inserters' queries +rather than idling their connections. It is not best-effort: one Datum the worker cannot convert fails the whole batch, because a substituted value is one the destination cannot tell from real data @@ -66,8 +67,8 @@ String PostGIS `geography` / `geometry` route to the oracle like any other extension type, but `render_ext_columns` (`src/ops/oracle.rs`) still claims 2-D points in-tree first, matched on `RelAttr.type_name` because their OIDs are dynamic. -A rendered `POINT(x y)` crosses as a literal cell and lands verbatim; anything -it does not claim crosses as on-disk bytes and gets `typoutput`. `ST_AsText` +A rendered `POINT(x y)` lands verbatim; anything it does not claim crosses as +on-disk bytes and gets `typoutput`. `ST_AsText` semantics are what the `String` mapping wants, and `typoutput` would give HEXEWKB instead @@ -124,8 +125,8 @@ all of that passes ## Splicing -The inserter decodes the response once with `clickhouse-c-rs`, then builds the -final block over three kinds of column: +The resolver decodes the response once with `clickhouse-c-rs`, then the +inserter builds the final block over three kinds of column: - local fixed / string / nullable slabs, through `build_leaf` + `build_root` - decoded oracle column trees, through `BlockBuilder::append_column` @@ -137,6 +138,20 @@ expected `TypeAst`. Nothing visits a value. The response block outlives the whole `send_with_retry` loop, so a ClickHouse reconnect resends the same columns rather than asking the oracle again +## Columns the request leaves out + +A column whose every cell is `Literal` or `Default`, against a `String` or +`Nullable(String)` target, is bytes the daemon rendered and ClickHouse wants +unchanged: the worker would wrap each in a `text` and pgch would cast it +straight back. `literal_column` builds those in the resolver, matching +`ws_append_default` on defaults — NULL under `Nullable`, empty string +otherwise. Anything wrapping a String (`Array`, `LowCardinality`) is PG's to +parse, and one un-rendered cell keeps its whole column in the request + +Live case is a PostGIS 2-D point column with a `String` target, which is +often a table's only oracle column — then the batch makes no request at all. +`walshadow_oracle_local_columns_total` counts columns taken this way + ## walshadow PG module Lives at [`pgext/`](../pgext/), built via PGXS. Not an extension: no control @@ -239,6 +254,26 @@ bound to the requested transaction. Losing the oid list loses the lock argument with it, which is why an empty list is only safe on the committed read — there is no tree to misattribute a writer to +## Worker pool + +Each worker serves one request per loop iteration, so one worker means +oracle throughput is one PG backend's conversion rate however wide the +daemon's inserter pool is. `walshadow.bridge_workers` (default 1, ceiling +`WS_MAX_WORKERS`) registers copies: worker 0 keeps the bare +`walshadow.socket_path`, worker `i` listens on `socket_path.i`. On a +daemon-owned shadow `--bridge-workers` writes that GUC, sizes the daemon's +socket pool to match, and with it the resolver pool + +`ENCODE_NATIVE` and `REPLAY_LSN` are stateless and read-only, so the pool +round-robins them. `HELLO` and `SCAN` pin to worker 0: a scan answers off +a replay position it reports back, and the first `HELLO` establishes the +identity every other worker is then checked against, so a build mismatch +across the pool fails closed at connect rather than mid-read + +`Bridge::connect_pooled` requires every configured slot to answer. A pool +short of its configured width is a half-started shadow, and running +narrower would hide it + ## Failure semantics Daemon requires worker at startup. `--bridge-socket` defaults to @@ -265,7 +300,13 @@ make a value depend on worker uptime Counters: `walshadow_oracle_{blocks,rows,cells}_total`, `walshadow_oracle_conversion_errors_total`, `walshadow_oracle_errors_total`. Native volume counts once, on the bridge, as -`walshadow_bridge_native_bytes_total` +`walshadow_bridge_native_bytes_total`. +`walshadow_bridge_lock_wait_seconds_total{op}` against +`walshadow_bridge_service_seconds_total{op}` is what says whether the +worker or the socket in front of it is the limiter, and +`walshadow_oracle_resolve_seconds_total` against +`walshadow_inserter_ch_seconds_total` which of the two overlapped stages +is ## Backpressure @@ -281,7 +322,8 @@ above `inline_value_max` so a toast value the pipeline admits always frames Removing per-row resolution from the decode pool removes every bridge await from decode workers: their throughput and ordering no longer depend on bridge -RTT +RTT. The resolver stage holds one extra batch per inserter beyond that, which +is what a queue deep enough to keep every inserter fed costs ## Pinning shadow locale diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index d049ce21..eb798b4f 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -27,7 +27,7 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{Context, Result}; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc; use tokio::task::JoinHandle; use tokio_postgres::Client; use walrus::pg::walparser::{Oid, RelFileNode}; @@ -38,10 +38,19 @@ use crate::backfill::backup_page_walk::{ use crate::backfill::backup_sink::{ CatalogFilenodes, DiskLanderSink, DiskLanderStats, MultiplexSink, }; -use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, StartInfo}; +use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, PumpStats, StartInfo}; use crate::decode::decoder_sink::TupleObserver; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; +/// Live counter handles, readable while the pump runs. The pump publishes +/// into these, so a caller ticks `/metrics` off them instead of waiting for +/// [`BootstrapOutcome`] +#[derive(Debug, Clone, Default)] +pub struct BootstrapProgress { + pub pump: Arc, + pub page_walk: Arc, +} + #[derive(Debug, Clone)] pub struct BootstrapConfig { /// Orchestrator pre-creates if missing. `Shadow::start` requires it @@ -51,6 +60,10 @@ pub struct BootstrapConfig { /// `rel_node < 16384` bootstrap rule misses. Empty in greenfield where /// seed runs after bootstrap pub catalog_filenodes: CatalogFilenodes, + /// Filenodes worth decoding: mapped relations plus their TOAST heaps. + /// `None` taps every seeded relation + pub tap_filenodes: Option>>, + pub progress: BootstrapProgress, } impl BootstrapConfig { @@ -58,6 +71,8 @@ impl BootstrapConfig { Self { shadow_data_dir, catalog_filenodes: CatalogFilenodes::new(), + tap_filenodes: None, + progress: BootstrapProgress::default(), } } @@ -65,14 +80,65 @@ impl BootstrapConfig { self.catalog_filenodes = c; self } + + /// Decline every filenode outside `set` at `begin`, so unmapped + /// relations drain off the wire without a page decode + pub fn with_tap_filenodes(mut self, set: Arc>) -> Self { + self.tap_filenodes = Some(set); + self + } } #[derive(Debug, Clone)] pub struct BootstrapOutcome { pub start: StartInfo, pub end: EndInfo, - pub disk: DiskLanderStats, - pub page_walk: PageWalkStats, + pub disk: Arc, + pub page_walk: Arc, + pub pump: Arc, +} + +/// Filenodes worth page-walking: relations `mapped` accepts, plus the TOAST +/// heap of each. `begin` declines everything else, so an unreplicated +/// relation's bytes drain off the wire without a page decode. +/// +/// `None` disables the filter: a mapped relation whose `reltoastrelid` the +/// seed did not resolve would otherwise have its TOAST heap skipped, and its +/// referring rows would fail resolution mid-drain. Fail open, walk everything. +pub fn tap_filenode_set( + catalog: &CatalogMap, + mapped: impl Fn(&RelName) -> bool, +) -> Option> { + use ahash::HashSetExt; + let by_oid: ahash::HashMap = catalog + .descriptors() + .map(|d| (d.oid, (d.rfn.db_node, d.rfn.rel_node))) + .collect(); + let mut set = ahash::HashSet::new(); + for d in catalog.descriptors() { + if !mapped(&d.rel_name) { + continue; + } + set.insert((d.rfn.db_node, d.rfn.rel_node)); + if d.toast_oid == 0 { + continue; + } + match by_oid.get(&d.toast_oid) { + Some(&key) => { + set.insert(key); + } + None => { + tracing::warn!( + target: "walshadow::bootstrap", + relation = %d.rel_name, + toast_oid = d.toast_oid, + "toast heap absent from the catalog seed, walking every relation", + ); + return None; + } + } + } + Some(set) } /// Spawn pump on a tokio task, yield `(rx, handle)` so caller drains @@ -102,12 +168,12 @@ pub fn spawn_greenfield_bootstrap( // count + skip (disabled mode, values NULL/default-filled). store_toast: bool, ) -> ( - mpsc::Receiver, + mpsc::Receiver>, JoinHandle>, ) { // Bounded: PageWalkSink::chunk awaits a free slot, so a slow drain // backpressures the source instead of buffering the whole relation - let (tx, rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (tx, rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); let pump = tokio::spawn(async move { tokio::fs::create_dir_all(&cfg.shadow_data_dir) .await @@ -119,46 +185,31 @@ pub fn spawn_greenfield_bootstrap( })?; let lander = DiskLanderSink::new(cfg.catalog_filenodes); - let page_walk = PageWalkSink::new(catalog_map, tx, store_toast); - let mux = MultiplexSink::new(lander, page_walk); - - // Keep typed Arc beside erased trait-object Arc to recover stats - // after source completes; both point at the same Mutex - let typed: Arc>> = Arc::new(Mutex::new(mux)); - let erased: Arc> = typed.clone(); + let disk = lander.stats.clone(); + let mut page_walk = PageWalkSink::new(catalog_map, tx, store_toast) + .with_stats(cfg.progress.page_walk.clone()); + if let Some(set) = cfg.tap_filenodes.clone() { + page_walk = page_walk.with_tap_filenodes(set); + } + // Counters are atomics on shared handles, so the sink never has to + // come back out of the source + let sink: Arc = Arc::new(MultiplexSink::new(lander, page_walk)); let data_dir = cfg.shadow_data_dir.clone(); let (start, end) = source - .run(data_dir, erased) + .run(data_dir, sink, cfg.progress.pump.clone()) .await .context("bootstrap: source.run")?; - // Source dropped its `erased` clone on return, so `try_unwrap` - // succeeds unless a clone leaked; else read through the Mutex - let outcome = match Arc::try_unwrap(typed) { - Ok(mtx) => { - let mux = mtx.into_inner(); - let (lander, page_walk) = mux.into_inner(); - // Dropping `page_walk` closes the channel sender in `out_tx`, - // so a concurrent drain observes channel-close on next recv - BootstrapOutcome { - start, - end, - disk: lander.stats, - page_walk: page_walk.stats, - } - } - Err(arc) => { - let g = arc.lock().await; - BootstrapOutcome { - start, - end, - disk: g.lander_stats().clone(), - page_walk: PageWalkStats::default(), - } - } - }; - Ok(outcome) + // Dropping the last sink Arc closes the channel sender in `out_tx`, + // so a concurrent drain observes channel-close on next recv + Ok(BootstrapOutcome { + start, + end, + disk, + page_walk: cfg.progress.page_walk.clone(), + pump: cfg.progress.pump.clone(), + }) }); (rx, pump) } @@ -176,8 +227,8 @@ pub async fn run_greenfield_bootstrap( let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); let drain = tokio::spawn(async move { let mut out = Vec::new(); - while let Some(t) = rx.recv().await { - out.push(t); + while let Some(slab) = rx.recv().await { + out.extend(slab); } out }); @@ -202,28 +253,30 @@ pub async fn run_greenfield_bootstrap( /// Final `on_xact_end` after channel-close lets the transitional emitter /// release CH state before the daemon swaps to the shadow-catalog emitter. pub async fn drain_backfill( - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver>, observer: &mut O, ) -> Result { let mut shipped: u64 = 0; let mut last_rfn: Option = None; let mut last_lsn: u64 = 0; - while let Some(tuple) = rx.recv().await { - if let Some(prev) = last_rfn - && prev != tuple.rfn - { - observer.on_xact_end(last_lsn).await.map_err(|e| { - anyhow::anyhow!("bootstrap drain: emitter rejected mid-table xact end: {e}") - })?; + while let Some(slab) = rx.recv().await { + for tuple in slab { + if let Some(prev) = last_rfn + && prev != tuple.rfn + { + observer.on_xact_end(last_lsn).await.map_err(|e| { + anyhow::anyhow!("bootstrap drain: emitter rejected mid-table xact end: {e}") + })?; + } + last_rfn = Some(tuple.rfn); + last_lsn = tuple.source_lsn; + let committed = tuple.into_committed_insert(); + observer + .on_tuple(&committed) + .await + .map_err(|e| anyhow::anyhow!("bootstrap drain: emitter rejected tuple: {e}"))?; + shipped += 1; } - last_rfn = Some(tuple.rfn); - last_lsn = tuple.source_lsn; - let committed = tuple.into_committed_insert(); - observer - .on_tuple(&committed) - .await - .map_err(|e| anyhow::anyhow!("bootstrap drain: emitter rejected tuple: {e}"))?; - shipped += 1; } observer .on_xact_end(last_lsn) @@ -424,27 +477,17 @@ mod tests { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { - { - let mut g = sink.lock().await; - g.start(&self.start).await?; - } - for (i, (meta, body)) in self.files.iter().enumerate() { + sink.start(&self.start).await?; + let target = + crate::backfill::backup_source::PumpTarget::new(data_dir, sink.clone(), stats); + for (meta, body) in self.files.iter() { let mut cur: &[u8] = body; - crate::backfill::backup_source::pump_entry( - &mut cur, - meta, - &data_dir, - &sink, - crate::backfill::backup_source::EntryId(i as u64), - ) - .await?; - } - { - let mut g = sink.lock().await; - g.finish(&self.end).await?; + crate::backfill::backup_source::pump_entry(&mut cur, meta, &target).await?; } + sink.finish(&self.end).await?; Ok((self.start.clone(), self.end.clone())) } } @@ -537,14 +580,14 @@ mod tests { async fn drain_backfill_synthesises_inserts_into_observer() { use crate::decode::decoder_sink::CollectingTupleObserver; - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); let rfn = RelFileNode { spc_node: 1663, db_node: 5, rel_node: 16400, }; for v in 0..3 { - tx.send(BackfillTuple { + tx.send(vec![BackfillTuple { rfn, xid: 100 + v, xmax: 0, @@ -555,7 +598,7 @@ mod tests { columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( v as i32, ))], - }) + }]) .await .unwrap(); } @@ -623,9 +666,9 @@ mod tests { db_node: 5, rel_node: 16400, }; - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); for v in 0..4u32 { - tx.send(BackfillTuple { + tx.send(vec![BackfillTuple { rfn, xid: v, xmax: 0, @@ -636,7 +679,7 @@ mod tests { columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( v as i32, ))], - }) + }]) .await .unwrap(); } @@ -654,7 +697,7 @@ mod tests { async fn drain_backfill_calls_on_xact_end_even_on_empty_channel() { // Sender dropped without a tuple; `on_xact_end` still fires so the // transitional emitter's INSERT cleanup runs unconditionally - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); drop(tx); let mut obs = CountingObserver::default(); let shipped = drain_backfill(rx, &mut obs).await.unwrap(); diff --git a/src/backfill/backup_backfill.rs b/src/backfill/backup_backfill.rs index fec9f6af..034a7403 100644 --- a/src/backfill/backup_backfill.rs +++ b/src/backfill/backup_backfill.rs @@ -55,7 +55,7 @@ use crate::backfill::backup_page_walk::{ BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTuple, CatalogMap, PageWalkSink, }; use crate::backfill::backup_sentinel::build_lsn_pair; -use crate::backfill::backup_source::{BackupSink, BackupSource}; +use crate::backfill::backup_source::{BackupSink, BackupSource, PumpStats}; use crate::backfill::backup_source_direct::DirectSource; use crate::backfill::backup_source_object_store::ObjectStoreSource; use crate::backfill::spool::{DEFERRED_SPOOL_MEM_MAX, DeferredSpool}; @@ -312,14 +312,15 @@ async fn walk_and_ship( let pg_xact = Arc::new(std::sync::Mutex::new(PgXactAccum::new())); let pg_multixact = Arc::new(std::sync::Mutex::new(PgMultiXactAccum::new())); - let (walk_tx, walk_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); - let (gated_tx, gated_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); - - let sink = PageWalkSink::new(filter.clone(), walk_tx, store_toast) - .with_pg_xact_accum(pg_xact.clone()) - .with_pg_multixact_accum(pg_multixact.clone()) - .with_lsn_overrides(lsn_overrides); - let erased: Arc> = Arc::new(Mutex::new(sink)); + let (walk_tx, walk_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (gated_tx, gated_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); + + let sink: Arc = Arc::new( + PageWalkSink::new(filter.clone(), walk_tx, store_toast) + .with_pg_xact_accum(pg_xact.clone()) + .with_pg_multixact_accum(pg_multixact.clone()) + .with_lsn_overrides(lsn_overrides), + ); // data_dir is never written: PageWalkSink only Taps/Skips let data_dir = ctx.scratch_dir.join("void"); @@ -360,7 +361,7 @@ async fn walk_and_ship( // against a complete pg_xact accum; a failed source drops the sender // and the gate discards them instead let run_res = source - .run(data_dir, erased) + .run(data_dir, sink, Arc::new(PumpStats::default())) .await .context("backup_backfill: source.run"); if run_res.is_ok() { @@ -469,8 +470,8 @@ struct GateStats { /// predates WAL coverage, skipping risks dropping a live row. #[allow(clippy::too_many_arguments)] async fn gate_task( - mut rx: mpsc::Receiver, - tx: mpsc::Sender, + mut rx: mpsc::Receiver>, + tx: mpsc::Sender>, filter: CatalogMap, pg_xact: Arc>, pg_multixact: Arc>, @@ -479,29 +480,33 @@ async fn gate_task( mut deferred: DeferredSpool, ) -> Result { let mut stats = GateStats::default(); - while let Some(t) = rx.recv().await { - if filter.is_toast(t.rfn.db_node, t.rfn.rel_node) { - if tx.send(t).await.is_err() { - return Ok(stats); + // Slab in, slab out: the gate never breaks a walk slab into per-tuple + // channel hops, it only drops what the visibility verdict removes + while let Some(slab) = rx.recv().await { + let mut pass = Vec::with_capacity(slab.len()); + for t in slab { + if filter.is_toast(t.rfn.db_node, t.rfn.rel_node) { + pass.push(t); + continue; } - continue; - } - match tuple_visibility(t.xid, t.xmax, t.infomask, None) { - Visibility::Emit => { - if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { - stats.multixact_emitted += 1; - } - stats.emitted += 1; - if tx.send(t).await.is_err() { - return Ok(stats); + match tuple_visibility(t.xid, t.xmax, t.infomask, None) { + Visibility::Emit => { + if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { + stats.multixact_emitted += 1; + } + stats.emitted += 1; + pass.push(t); } + Visibility::Skip => stats.gated += 1, + Visibility::Defer => deferred + .push(t) + .await + .map_err(|e| format!("backup_backfill: deferred spool: {e}"))?, + Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), } - Visibility::Skip => stats.gated += 1, - Visibility::Defer => deferred - .push(t) - .await - .map_err(|e| format!("backup_backfill: deferred spool: {e}"))?, - Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), + } + if !pass.is_empty() && tx.send(pass).await.is_err() { + return Ok(stats); } } // Walk EOF: sink dropped. Deferred tuples sit in the spool past its @@ -535,7 +540,7 @@ async fn gate_task( stats.multixact_emitted += 1; } stats.emitted += 1; - if tx.send(t).await.is_err() { + if tx.send(vec![t]).await.is_err() { return Ok(stats); } } @@ -1603,30 +1608,30 @@ mod tests { // Hinted-committed: passes through immediately walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID, - )) + )]) .await .unwrap(); // Hinted-aborted: gated walk_tx - .send(tuple(16400, 101, 0, HEAP_XMIN_INVALID)) + .send(vec![tuple(16400, 101, 0, HEAP_XMIN_INVALID)]) .await .unwrap(); // Unhinted, gap-committed writer: deferred, then emitted via patch - walk_tx.send(tuple(16400, 500, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 500, 0, 0)]).await.unwrap(); // Unhinted, gap-aborted writer: deferred, then gated via patch - walk_tx.send(tuple(16400, 600, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 600, 0, 0)]).await.unwrap(); drop(walk_tx); walk_ok_tx.send(()).unwrap(); let stats = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { - got.push(t.xid); + got.extend(t.iter().map(|x| x.xid)); } assert_eq!(got, [100, 500]); assert_eq!(stats.emitted, 2); @@ -1667,23 +1672,23 @@ mod tests { // Hinted-committed: routed before the failure, stays flushed walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID, - )) + )]) .await .unwrap(); // Unhinted: deferred, must be discarded - walk_tx.send(tuple(16400, 500, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 500, 0, 0)]).await.unwrap(); drop(walk_tx); drop(walk_ok_tx); let stats = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { - got.push(t.xid); + got.extend(t.iter().map(|x| x.xid)); } assert_eq!(got, [100], "deferred tuple not emitted"); assert_eq!(stats.emitted, 1); @@ -1732,12 +1737,12 @@ mod tests { )); walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 10, HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI, - )) + )]) .await .unwrap(); drop(walk_tx); @@ -1772,12 +1777,12 @@ mod tests { )); walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 10, HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI, - )) + )]) .await .unwrap(); drop(walk_tx); diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index 8e697228..dda657ca 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -14,6 +14,8 @@ use std::io; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; use async_trait::async_trait; use thiserror::Error; @@ -21,14 +23,14 @@ use tokio::sync::mpsc; use walrus::pg::walparser::{Oid, RelFileNode}; use crate::backfill::backup_source::{ - BackupSink, EndInfo, EntryId, FileAction, FileKind, FileMeta, StartInfo, + BackupSink, EntrySink, FileAction, FileKind, FileMeta, StartInfo, }; use crate::backfill::pg_path::{BaseRelFile, RelFork, parse_base_path}; use crate::decode::heap_decoder::{ ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, decode_block_data, }; use crate::schema::RelDescriptor; -use ahash::{HashMap, HashMapExt}; +use ahash::{HashMap, HashMapExt, HashSet}; /// Heap page size, PG compile-time, identical to wal-rus `BLOCK_SIZE` pub const PAGE_BYTES: usize = 8192; @@ -45,11 +47,16 @@ pub const LP_NORMAL: u8 = 1; /// `pg_toast` regnamespace; TOAST tables ship as `pg_toast_` pub const PG_TOAST_NS: &str = "pg_toast"; -/// Bootstrap tuple channel depth. Small + bounded so a saturated CH -/// inserter parks the page walk (and its source fetch) rather than -/// buffering a whole relation in RAM. Just deep enough to absorb a -/// page's worth of tuples without thrashing wakeups. -pub const BOOTSTRAP_TUPLE_CHANNEL_CAP: usize = 256; +/// Body bytes a segment accumulates before its complete pages walk. Sets +/// the `spawn_blocking` grain and, with the channel depth, the pages in +/// flight; 16 pages decode in well under a millisecond, so the hop cost +/// amortizes without pinning much heap. +pub const SLAB_BYTES: usize = 16 * PAGE_BYTES; +/// Bootstrap tuple channel depth, in slabs. Small + bounded so a saturated +/// CH inserter parks the page walk (and its source fetch) rather than +/// buffering a whole relation in RAM. `SLAB_BYTES * CAP` bounds the heap +/// payload in flight per concurrent segment. +pub const BOOTSTRAP_TUPLE_CHANNEL_CAP: usize = 8; #[derive(Debug, Error)] pub enum PageWalkError { @@ -155,22 +162,62 @@ impl CatalogMap { } } -/// Per-pump counters, operator-visible -#[derive(Debug, Default, Clone)] -pub struct PageWalkStats { - pub files_seen: u64, - pub files_walked: u64, - /// Filenode absent from catalog map, typically a race against the seed - pub files_skipped_unknown_filenode: u64, - pub toast_files_observed: u64, +crate::atomic_stats! { + /// Per-pump counters, operator-visible. Atomic so per-entry walkers + /// share one handle instead of funnelling through the sink lock + pub struct PageWalkStats { + pub files_seen, + pub files_walked, + /// Filenode absent from catalog map, typically a race against the seed + pub files_skipped_unknown_filenode, + /// Filenode outside the mapped set, declined before any page decode + pub files_skipped_unmapped, + pub toast_files_observed, + pub pages_walked, + pub slots_seen, + pub tuples_emitted, + pub tuples_skipped_lp_flag, + pub tuples_skipped_truncated, + /// Trailing partial-page bytes; PG heap files are page-aligned so + /// nonzero is anomalous + pub tail_bytes_dropped, + /// `PageWalker::walk_page` CPU, decode included + pub decode_nanos, + /// Blocked on a free tuple-channel slot: the bootstrap + /// backpressure point, so this is emitter drain time seen by the walk + pub channel_block_nanos, + } +} + +/// Plain tally one `walk_page` fills, published into [`PageWalkStats`] once +/// per slab. Per-tuple `fetch_add` on a line the reader task also touches +/// (the shared `Arc`'s refcount sits in the same allocation) costs more +/// than the decode it counts +#[derive(Debug, Default, Clone, Copy)] +pub struct PageWalkTally { pub pages_walked: u64, pub slots_seen: u64, pub tuples_emitted: u64, pub tuples_skipped_lp_flag: u64, pub tuples_skipped_truncated: u64, - /// Trailing partial-page bytes; PG heap files are page-aligned so - /// nonzero is anomalous - pub tail_bytes_dropped: u64, +} + +impl PageWalkTally { + pub fn publish(&self, stats: &PageWalkStats) { + let add = |c: &AtomicU64, v: u64| { + if v > 0 { + c.fetch_add(v, Ordering::Relaxed); + } + }; + add(&stats.pages_walked, self.pages_walked); + add(&stats.slots_seen, self.slots_seen); + add(&stats.tuples_emitted, self.tuples_emitted); + add(&stats.tuples_skipped_lp_flag, self.tuples_skipped_lp_flag); + add( + &stats.tuples_skipped_truncated, + self.tuples_skipped_truncated, + ); + } } /// Walks one 8 KiB page, emitting `BackfillTuple`s for `LP_NORMAL` @@ -191,7 +238,7 @@ impl<'a> PageWalker<'a> { page: &[u8], block_no: u32, out: &mut Vec, - stats: &mut PageWalkStats, + tally: &mut PageWalkTally, ) -> Result<(), PageWalkError> { if page.len() < SIZE_OF_PAGE_HEADER { return Err(PageWalkError::BadPageHeader { @@ -207,12 +254,12 @@ impl<'a> PageWalker<'a> { let pd_lower = u16::from_le_bytes(page[12..14].try_into().unwrap()); let pd_upper = u16::from_le_bytes(page[14..16].try_into().unwrap()); if pd_upper == 0 && page.iter().all(|&byte| byte == 0) { - stats.pages_walked += 1; + tally.pages_walked += 1; return Ok(()); } if pd_lower as usize == SIZE_OF_PAGE_HEADER && pd_upper as usize == PAGE_BYTES { // Fresh / empty page - stats.pages_walked += 1; + tally.pages_walked += 1; return Ok(()); } if (pd_lower as usize) < SIZE_OF_PAGE_HEADER @@ -226,9 +273,10 @@ impl<'a> PageWalker<'a> { }); } let n_slots = (pd_lower as usize - SIZE_OF_PAGE_HEADER) / SIZE_OF_ITEM_ID; - stats.pages_walked += 1; + tally.pages_walked += 1; + tally.slots_seen += n_slots as u64; + out.reserve(n_slots); for i in 0..n_slots { - stats.slots_seen += 1; let off = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; let raw = u32::from_le_bytes(page[off..off + 4].try_into().unwrap()); // bit-packed: lp_off (15) | lp_flags (2) | lp_len (15) @@ -236,11 +284,11 @@ impl<'a> PageWalker<'a> { let lp_flags = ((raw >> 15) & 0x3) as u8; let lp_len = ((raw >> 17) & 0x7FFF) as usize; if lp_flags != LP_NORMAL { - stats.tuples_skipped_lp_flag += 1; + tally.tuples_skipped_lp_flag += 1; continue; } if lp_off + lp_len > PAGE_BYTES || lp_len == 0 { - stats.tuples_skipped_truncated += 1; + tally.tuples_skipped_truncated += 1; continue; } let tuple_bytes = &page[lp_off..lp_off + lp_len]; @@ -257,9 +305,9 @@ impl<'a> PageWalker<'a> { offnum: (i + 1) as u16, columns, }); - stats.tuples_emitted += 1; + tally.tuples_emitted += 1; } - None => stats.tuples_skipped_truncated += 1, + None => tally.tuples_skipped_truncated += 1, } } Ok(()) @@ -348,29 +396,30 @@ pub(crate) fn decode_on_page_tuple( } } -/// Tap sink that buffers tar entry bytes and walks them 8 KiB at a -/// time, shipping tuples over `out_tx` to an async drain task. +/// Tap router: resolves a segment's descriptor at `begin` and hands back a +/// [`PageWalkEntry`] that owns the walk. Nothing here is on the body path, +/// so concurrent tar parts never queue behind each other. pub struct PageWalkSink { catalog: CatalogMap, - source_lsn: u64, + /// `StartInfo.start_lsn`, set once before any `begin` + source_lsn: AtomicU64, /// Per-rfn `_lsn` tag overriding `source_lsn` (backup-sourced opt-in /// backfills tag each rel with its own boundary; greenfield leaves /// this empty). Keyed `(db_node, rel_node)`. lsn_overrides: HashMap<(Oid, Oid), u64>, - pub stats: PageWalkStats, - /// Bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]): `chunk` is async, so a - /// full channel awaits in `ship_tuple`, parking the source body read + pub stats: Arc, + /// Bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]) slab channel: a full + /// channel awaits in the entry sink, parking that entry's body read /// instead of buffering. Backpressure, not a buffer. - out_tx: Option>, + out_tx: Option>>, /// Test-only capture, populated when `out_tx` is None - pub captured: Vec, - /// Per-entry state keyed by `EntryId`. A map, not one slot, because - /// object_store fan-out interleaves begin/chunk across concurrent - /// parts; one slot would let part B's begin clobber part A's - /// in-flight entry, mis-framing pages and misattributing rfns. - cur: HashMap, + captured: Arc>>, /// Decode TOAST pages only when configured store consumes them store_toast: bool, + /// Filenodes worth decoding, mapped relations plus their TOAST heaps. + /// `None` taps everything the catalog map holds. A superset filter, not a + /// second source of truth: the drain's `skip_initial` stays the authority + tap_filenodes: Option>>, /// `pg_xact/` segments Tap into here for backfill visibility gate /// (plans/add_table.md); `None` (greenfield) keeps Skip. pg_xact: Option>>, @@ -386,36 +435,21 @@ enum SlruSegment { MultiMembers(u32), } -struct TapEntry { - /// Block number within the relation, global across `.N` segments - /// (seeded `segno * RELSEG_BLOCKS`): TOAST rows key on - /// `(blkno, offnum)`, and all walk rows share one LSN, so per-file - /// numbering would collide segment TIDs at equal version - block_no: u32, - /// Carry-over bytes when a chunk read straddled a page boundary - page_buf: Vec, - is_toast: bool, - desc: Option>, - /// `Some` for SLRU entries (`pg_xact/`, `pg_multixact/`): bytes - /// accumulate whole (no page framing) and install at `end()` - slru: Option, -} - impl PageWalkSink { pub fn new( catalog: CatalogMap, - out_tx: mpsc::Sender, + out_tx: mpsc::Sender>, store_toast: bool, ) -> Self { Self { catalog, - source_lsn: 0, + source_lsn: AtomicU64::new(0), lsn_overrides: HashMap::new(), - stats: PageWalkStats::default(), + stats: Arc::new(PageWalkStats::default()), out_tx: Some(out_tx), - captured: Vec::new(), - cur: HashMap::new(), + captured: Arc::default(), store_toast, + tap_filenodes: None, pg_xact: None, pg_multixact: None, } @@ -439,6 +473,20 @@ impl PageWalkSink { self } + /// Decline filenodes outside `set` at `begin`. Bytes still drain off the + /// wire; tuples never decode + pub fn with_tap_filenodes(mut self, set: Arc>) -> Self { + self.tap_filenodes = Some(set); + self + } + + /// Share the counter handle so a caller watches the walk live instead + /// of waiting for the pump to hand its sink back. + pub fn with_stats(mut self, stats: Arc) -> Self { + self.stats = stats; + self + } + /// Tag listed rfns' rows with their own `_lsn` instead of `source_lsn`. pub fn with_lsn_overrides(mut self, overrides: HashMap<(Oid, Oid), u64>) -> Self { self.lsn_overrides = overrides; @@ -450,13 +498,13 @@ impl PageWalkSink { pub fn new_capturing(catalog: CatalogMap) -> Self { Self { catalog, - source_lsn: 0, + source_lsn: AtomicU64::new(0), lsn_overrides: HashMap::new(), - stats: PageWalkStats::default(), + stats: Arc::new(PageWalkStats::default()), out_tx: None, - captured: Vec::new(), - cur: HashMap::new(), + captured: Arc::default(), store_toast: false, + tap_filenodes: None, pg_xact: None, pg_multixact: None, } @@ -471,8 +519,14 @@ impl PageWalkSink { } } + /// Tuples a capturing sink collected. Empty once `out_tx` is wired + #[cfg(test)] + pub fn captured(&self) -> Vec { + self.captured.lock().expect("captured lock").clone() + } + pub fn source_lsn(&self) -> u64 { - self.source_lsn + self.source_lsn.load(Ordering::Relaxed) } fn classify(&self, meta: &FileMeta) -> Option { @@ -497,91 +551,25 @@ impl PageWalkSink { } None } - - async fn flush_full_pages(&mut self, id: EntryId) -> io::Result<()> { - loop { - // Take the page so the entry borrow drops before touching - // self.stats / out_tx.send - let (block_no, is_toast, desc_opt, page) = { - let Some(entry) = self.cur.get_mut(&id) else { - return Ok(()); - }; - if entry.slru.is_some() || entry.page_buf.len() < PAGE_BYTES { - return Ok(()); - } - let block_no = entry.block_no; - entry.block_no = entry.block_no.saturating_add(1); - let page: Vec = entry.page_buf.drain(..PAGE_BYTES).collect(); - (block_no, entry.is_toast, entry.desc.clone(), page) - }; - - if is_toast && !self.store_toast { - self.stats.pages_walked += 1; - continue; - } - let Some(desc) = desc_opt else { - // Filenode absent from seed; stats counted at begin() - continue; - }; - let lsn = self - .lsn_overrides - .get(&(desc.rfn.db_node, desc.rfn.rel_node)) - .copied() - .unwrap_or(self.source_lsn); - let walker = PageWalker::new(&desc, lsn); - let mut local_out = Vec::new(); - if let Err(e) = walker.walk_page(&page, block_no, &mut local_out, &mut self.stats) { - tracing::warn!( - target = "walshadow::backup_page_walk", - block = block_no, - error = %e, - "page walk skipped due to framing error" - ); - continue; - } - for t in local_out { - self.ship_tuple(t).await?; - } - } - } - - async fn ship_tuple(&mut self, t: BackfillTuple) -> io::Result<()> { - match &self.out_tx { - // Awaits a free slot when full: this is the bootstrap - // backpressure point, parking the source until CH drains - Some(tx) => tx.send(t).await.map_err(|e| { - io::Error::other(format!("PageWalkSink: emitter channel closed: {e}")) - }), - None => { - self.captured.push(t); - Ok(()) - } - } - } } #[async_trait] impl BackupSink for PageWalkSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { - self.source_lsn = info.start_lsn; + async fn start(&self, info: &StartInfo) -> io::Result<()> { + self.source_lsn.store(info.start_lsn, Ordering::Relaxed); Ok(()) } - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result { + async fn begin(&self, meta: &FileMeta) -> io::Result { if matches!(meta.kind, FileKind::File) && let Some(slru) = self.classify_slru(&meta.path) { - self.cur.insert( - entry, - TapEntry { - block_no: 0, - page_buf: Vec::with_capacity(meta.size as usize), - is_toast: false, - desc: None, - slru: Some(slru), - }, - ); - return Ok(FileAction::Tap); + return Ok(FileAction::Tap(Box::new(SlruEntry { + seg: slru, + buf: Vec::with_capacity(meta.size as usize), + pg_xact: self.pg_xact.clone(), + pg_multixact: self.pg_multixact.clone(), + }))); } let Some(f) = self.classify(meta) else { // Not base//; multiplex sink falls back to lander @@ -591,86 +579,281 @@ impl BackupSink for PageWalkSink { // fsm/vm carry no tuples; keep them out of the TID-producing walk return Ok(FileAction::Skip); } - self.stats.files_seen += 1; + self.stats.files_seen.fetch_add(1, Ordering::Relaxed); + // Mapping filter first: an unmapped relation would decode every page + // only for the drain to discard it into `unsupported_relations` + if self + .tap_filenodes + .as_ref() + .is_some_and(|set| !set.contains(&(f.db, f.filenode))) + { + self.stats + .files_skipped_unmapped + .fetch_add(1, Ordering::Relaxed); + return Ok(FileAction::Skip); + } let desc = self.catalog.get(f.db, f.filenode); let is_toast = self.catalog.is_toast(f.db, f.filenode); if is_toast { - self.stats.toast_files_observed += 1; + self.stats + .toast_files_observed + .fetch_add(1, Ordering::Relaxed); } else if desc.is_some() { - self.stats.files_walked += 1; + self.stats.files_walked.fetch_add(1, Ordering::Relaxed); } else { // Filenode absent from map: seed race (greenfield) or non-opted // rel (filtered backfill pass, where this is most files). Skip // drains body without page buffering; mux honours the decline - self.stats.files_skipped_unknown_filenode += 1; + self.stats + .files_skipped_unknown_filenode + .fetch_add(1, Ordering::Relaxed); return Ok(FileAction::Skip); } - self.cur.insert( - entry, - TapEntry { - block_no: f.segno.saturating_mul(RELSEG_BLOCKS), - page_buf: Vec::with_capacity(PAGE_BYTES * 2), - is_toast, - desc, - slru: None, + let Some(desc) = desc else { + // TOAST heap whose descriptor the map lacks; count pages, no walk + return Ok(FileAction::Tap(Box::new(PageWalkEntry::counting( + f.segno.saturating_mul(RELSEG_BLOCKS), + self.stats.clone(), + )))); + }; + let lsn = self + .lsn_overrides + .get(&(desc.rfn.db_node, desc.rfn.rel_node)) + .copied() + .unwrap_or_else(|| self.source_lsn()); + Ok(FileAction::Tap(Box::new(PageWalkEntry { + block_no: f.segno.saturating_mul(RELSEG_BLOCKS), + slab: Vec::with_capacity(SLAB_BYTES + PAGE_BYTES), + spare: None, + walk: (!is_toast || self.store_toast).then_some(WalkTarget { desc, lsn }), + out: match &self.out_tx { + Some(tx) => Out::Channel(tx.clone()), + None => Out::Captured(self.captured.clone()), }, - ); - Ok(FileAction::Tap) + stats: self.stats.clone(), + pending: None, + }))) } +} + +/// Descriptor a slab walks against. `None` on a TOAST heap the configured +/// store does not consume: its pages are counted, never decoded +struct WalkTarget { + desc: Arc, + lsn: u64, +} + +enum Out { + Channel(mpsc::Sender>), + Captured(Arc>>), +} + +/// One user-heap segment's walk. Owns its slab, so page framing and decode +/// run without touching the router; the walk itself goes to `spawn_blocking` +/// so tar read and decompress overlap it. +pub struct PageWalkEntry { + /// Block number within the relation, global across `.N` segments + /// (seeded `segno * RELSEG_BLOCKS`): TOAST rows key on + /// `(blkno, offnum)`, and all walk rows share one LSN, so per-file + /// numbering would collide segment TIDs at equal version + block_no: u32, + /// Accumulates body bytes; complete pages walk in place out of it and + /// only a sub-page remainder carries into the next slab + slab: Vec, + /// The slab last handed to the walk, back for reuse. Two buffers + /// ping-pong, so a segment allocates twice however long it is + spare: Option>, + walk: Option, + out: Out, + stats: Arc, + /// Previous slab's walk, still running. One slab of overlap: the tar + /// reader refills while the blocking pool decodes, so read and decode + /// cost `max`, not `sum` + pending: Option, Vec)>>, +} - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()> { - match self.cur.get_mut(&entry) { - Some(e) => e.page_buf.extend_from_slice(bytes), - None => return Err(io::Error::other("PageWalkSink: chunk before begin")), +impl PageWalkEntry { + /// Counts pages without decoding them + fn counting(block_no: u32, stats: Arc) -> Self { + Self { + block_no, + slab: Vec::with_capacity(SLAB_BYTES + PAGE_BYTES), + spare: None, + walk: None, + out: Out::Captured(Arc::default()), + stats, + pending: None, } - self.flush_full_pages(entry).await?; - Ok(()) } - async fn end(&mut self, entry: EntryId) -> io::Result<()> { - if let Some(e) = self.cur.remove(&entry) { - match e.slru { - Some(SlruSegment::PgXact(segno)) => { - if let Some(accum) = &self.pg_xact { - accum - .lock() - .expect("pg_xact accum lock") - .insert_segment(segno, e.page_buf); - } - return Ok(()); - } - Some(SlruSegment::MultiOffsets(segno)) => { - if let Some(accum) = &self.pg_multixact { - accum - .lock() - .expect("pg_multixact accum lock") - .insert_offsets_segment(segno, e.page_buf); - } - return Ok(()); - } - Some(SlruSegment::MultiMembers(segno)) => { - if let Some(accum) = &self.pg_multixact { - accum - .lock() - .expect("pg_multixact accum lock") - .insert_members_segment(segno, e.page_buf); - } - return Ok(()); + /// Collect the in-flight walk: recover its slab and ship its tuples. + /// Shipping here is what carries emitter backpressure back to the read + async fn join_pending(&mut self) -> io::Result<()> { + let Some(handle) = self.pending.take() else { + return Ok(()); + }; + let (walked, tuples) = handle.await.map_err(io::Error::other)?; + self.spare = Some(walked); + self.ship(tuples).await + } + + /// Hand every complete page in the slab to the walk, carrying the + /// sub-page remainder into the next slab. + async fn drain_slab(&mut self) -> io::Result<()> { + let full = self.slab.len() / PAGE_BYTES; + if full == 0 { + return Ok(()); + } + let take = full * PAGE_BYTES; + let first_block = self.block_no; + self.block_no = self.block_no.saturating_add(full as u32); + + let Some(target) = self.walk.as_ref() else { + self.stats + .pages_walked + .fetch_add(full as u64, Ordering::Relaxed); + self.slab.drain(..take); + return Ok(()); + }; + let desc = target.desc.clone(); + let lsn = target.lsn; + + // At most one walk in flight, so a segment holds two slabs, never more + self.join_pending().await?; + let mut next = self + .spare + .take() + .unwrap_or_else(|| Vec::with_capacity(SLAB_BYTES + PAGE_BYTES)); + next.clear(); + next.extend_from_slice(&self.slab[take..]); + let mut walked = std::mem::replace(&mut self.slab, next); + walked.truncate(take); + + let stats = self.stats.clone(); + // Pure CPU over a byte slice; off the async runtime so the drain + // stage it feeds keeps a worker to itself + self.pending = Some(tokio::task::spawn_blocking(move || { + let walker = PageWalker::new(&desc, lsn); + let mut out = Vec::new(); + let mut tally = PageWalkTally::default(); + let started = Instant::now(); + for i in 0..full { + let page = &walked[i * PAGE_BYTES..(i + 1) * PAGE_BYTES]; + let block = first_block.saturating_add(i as u32); + if let Err(e) = walker.walk_page(page, block, &mut out, &mut tally) { + tracing::warn!( + target = "walshadow::backup_page_walk", + block, + error = %e, + "page walk skipped due to framing error" + ); } - None => {} } - let trailing = e.page_buf.len() as u64; - if trailing > 0 { - // PG heap files are page-aligned; trailing bytes are - // zero-padding or anomalous, so count without decoding - self.stats.tail_bytes_dropped += trailing; + tally.publish(&stats); + stats + .decode_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + (walked, out) + })); + Ok(()) + } + + async fn ship(&self, tuples: Vec) -> io::Result<()> { + if tuples.is_empty() { + return Ok(()); + } + match &self.out { + // Awaits a free slot when full: this is the bootstrap + // backpressure point, parking this segment until CH drains + Out::Channel(tx) => { + let started = Instant::now(); + let res = tx.send(tuples).await.map_err(|e| { + io::Error::other(format!("PageWalkSink: emitter channel closed: {e}")) + }); + self.stats + .channel_block_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + res + } + Out::Captured(buf) => { + buf.lock().expect("captured lock").extend(tuples); + Ok(()) } } + } +} + +#[async_trait] +impl EntrySink for PageWalkEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.slab.extend_from_slice(bytes); + if self.slab.len() >= SLAB_BYTES { + self.drain_slab().await?; + } + Ok(()) + } + + async fn end(mut self: Box) -> io::Result<()> { + self.drain_slab().await?; + self.join_pending().await?; + let trailing = self.slab.len() as u64; + if trailing > 0 { + // PG heap files are page-aligned; trailing bytes are + // zero-padding or anomalous, so count without decoding + self.stats + .tail_bytes_dropped + .fetch_add(trailing, Ordering::Relaxed); + } + Ok(()) + } +} + +/// `pg_xact` / `pg_multixact` segment: no page framing, bytes accumulate +/// whole and install into the visibility accum at `end` +struct SlruEntry { + seg: SlruSegment, + buf: Vec, + pg_xact: Option>>, + pg_multixact: Option>>, +} + +#[async_trait] +impl EntrySink for SlruEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.buf.extend_from_slice(bytes); Ok(()) } - async fn finish(&mut self, _info: &EndInfo) -> io::Result<()> { - // Channel close happens when caller drops this sink's Arc + async fn end(self: Box) -> io::Result<()> { + let SlruEntry { + seg, + buf, + pg_xact, + pg_multixact, + } = *self; + match seg { + SlruSegment::PgXact(segno) => { + if let Some(a) = pg_xact { + a.lock() + .expect("pg_xact accum lock") + .insert_segment(segno, buf); + } + } + SlruSegment::MultiOffsets(segno) => { + if let Some(a) = pg_multixact { + a.lock() + .expect("pg_multixact accum lock") + .insert_offsets_segment(segno, buf); + } + } + SlruSegment::MultiMembers(segno) => { + if let Some(a) = pg_multixact { + a.lock() + .expect("pg_multixact accum lock") + .insert_members_segment(segno, buf); + } + } + } Ok(()) } } @@ -737,12 +920,35 @@ pub(crate) fn synth_single_tuple_page(value: i32) -> [u8; PAGE_BYTES] { #[cfg(test)] mod tests { use super::*; + + use crate::backfill::backup_source::EndInfo; + + fn ld(a: &AtomicU64) -> u64 { + a.load(Ordering::Relaxed) + } + + /// `begin` must tap; hands back the owned entry sink + async fn tap(sink: &PageWalkSink, meta: &FileMeta) -> Box { + match sink.begin(meta).await.unwrap() { + FileAction::Tap(e) => e, + other => panic!("expected Tap for {}, got {other:?}", meta.path.display()), + } + } + + fn heap_meta(path: &str) -> FileMeta { + FileMeta { + path: PathBuf::from(path), + size: PAGE_BYTES as u64, + mode: 0o600, + kind: FileKind::File, + } + } use crate::schema::RelName; use std::path::PathBuf; #[tokio::test] async fn source_lsn_reflects_start_info() { - let mut sink = PageWalkSink::new_capturing(CatalogMap::new()); + let sink = PageWalkSink::new_capturing(CatalogMap::new()); assert_eq!(sink.source_lsn(), 0); sink.start(&StartInfo { start_lsn: 0xABCD_1234, @@ -760,15 +966,15 @@ mod tests { let walker = PageWalker::new(&rel, 0xABCD); let page = synth_single_tuple_page(42); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert_eq!(out.len(), 1); assert_eq!(out[0].source_lsn, 0xABCD); assert_eq!(out[0].xid, 99); assert_eq!(out[0].columns.len(), 1); assert!(matches!(out[0].columns[0], Some(ColumnValue::Int4(42)))); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.tuples_emitted, 1); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.tuples_emitted, 1); } #[test] @@ -780,11 +986,11 @@ mod tests { page[12..14].copy_from_slice(&(SIZE_OF_PAGE_HEADER as u16).to_le_bytes()); page[14..16].copy_from_slice(&(PAGE_BYTES as u16).to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert!(out.is_empty()); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.slots_seen, 0); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.slots_seen, 0); } #[test] @@ -792,13 +998,13 @@ mod tests { let rel = make_rel(); let walker = PageWalker::new(&rel, 0); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); + let mut tally = PageWalkTally::default(); walker - .walk_page(&[0; PAGE_BYTES], 0, &mut out, &mut stats) + .walk_page(&[0; PAGE_BYTES], 0, &mut out, &mut tally) .unwrap(); assert!(out.is_empty()); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.slots_seen, 0); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.slots_seen, 0); } #[test] @@ -817,11 +1023,11 @@ mod tests { let new_raw = lp_off | (3u32 << 15) | (lp_len << 17); page[SIZE_OF_PAGE_HEADER..SIZE_OF_PAGE_HEADER + 4].copy_from_slice(&new_raw.to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert!(out.is_empty()); - assert_eq!(stats.tuples_skipped_lp_flag, 1); - assert_eq!(stats.tuples_emitted, 0); + assert_eq!(tally.tuples_skipped_lp_flag, 1); + assert_eq!(tally.tuples_emitted, 0); } #[test] @@ -833,8 +1039,8 @@ mod tests { page[12..14].copy_from_slice(&(PAGE_BYTES as u16).to_le_bytes()); page[14..16].copy_from_slice(&(SIZE_OF_PAGE_HEADER as u16).to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - let err = walker.walk_page(&page, 0, &mut out, &mut stats); + let mut tally = PageWalkTally::default(); + let err = walker.walk_page(&page, 0, &mut out, &mut tally); assert!(matches!(err, Err(PageWalkError::BadPageHeader { .. }))); } @@ -860,7 +1066,7 @@ mod tests { async fn pagewalk_sink_decodes_one_page_via_chunk_stream() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1234_5678, timeline: 1, @@ -868,20 +1074,12 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(0); - let action = sink.begin(id, &meta).await.unwrap(); - assert_eq!(action, FileAction::Tap); + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; let page = synth_single_tuple_page(99); // Two chunks exercise the buffer-across-chunk path - sink.chunk(id, &page[..4096]).await.unwrap(); - sink.chunk(id, &page[4096..]).await.unwrap(); - sink.end(id).await.unwrap(); + entry.chunk(&page[..4096]).await.unwrap(); + entry.chunk(&page[4096..]).await.unwrap(); + entry.end().await.unwrap(); sink.finish(&EndInfo { end_lsn: 0, timeline: 1, @@ -889,17 +1087,18 @@ mod tests { .await .unwrap(); - assert_eq!(sink.captured.len(), 1); - assert_eq!(sink.captured[0].source_lsn, 0x1234_5678); - assert_eq!(sink.captured[0].xid, 99); + let captured = sink.captured(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].source_lsn, 0x1234_5678); + assert_eq!(captured[0].xid, 99); assert!(matches!( - sink.captured[0].columns[0], + captured[0].columns[0], Some(ColumnValue::Int4(99)) )); - assert_eq!(sink.stats.files_seen, 1); - assert_eq!(sink.stats.files_walked, 1); - assert_eq!(sink.stats.pages_walked, 1); - assert_eq!(sink.stats.tuples_emitted, 1); + assert_eq!(ld(&sink.stats.files_seen), 1); + assert_eq!(ld(&sink.stats.files_walked), 1); + assert_eq!(ld(&sink.stats.pages_walked), 1); + assert_eq!(ld(&sink.stats.tuples_emitted), 1); } /// `.N` segment tuples get global block numbers: same local page + @@ -909,7 +1108,7 @@ mod tests { async fn pagewalk_sink_seeds_segment_block_numbers() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -917,26 +1116,20 @@ mod tests { }) .await .unwrap(); - for (i, path) in ["base/5/16400", "base/5/16400.1"].iter().enumerate() { - let meta = FileMeta { - path: PathBuf::from(path), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(i as u64); - assert_eq!(sink.begin(id, &meta).await.unwrap(), FileAction::Tap); - sink.chunk(id, &synth_single_tuple_page(7)).await.unwrap(); - sink.end(id).await.unwrap(); + for path in ["base/5/16400", "base/5/16400.1"] { + let mut entry = tap(&sink, &heap_meta(path)).await; + entry.chunk(&synth_single_tuple_page(7)).await.unwrap(); + entry.end().await.unwrap(); } - assert_eq!(sink.captured.len(), 2); + let captured = sink.captured(); + assert_eq!(captured.len(), 2); assert_eq!( - (sink.captured[0].blkno, sink.captured[0].offnum), + (captured[0].blkno, captured[0].offnum), (0, 1), "base file walks from block 0" ); assert_eq!( - (sink.captured[1].blkno, sink.captured[1].offnum), + (captured[1].blkno, captured[1].offnum), (RELSEG_BLOCKS, 1), "segment 1 walks from its global block" ); @@ -947,7 +1140,7 @@ mod tests { async fn pagewalk_sink_skips_non_main_forks() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -956,19 +1149,15 @@ mod tests { .await .unwrap(); for path in ["base/5/16400_fsm", "base/5/16400_vm", "base/5/16400_vm.1"] { - let meta = FileMeta { - path: PathBuf::from(path), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - assert_eq!( - sink.begin(EntryId(9), &meta).await.unwrap(), - FileAction::Skip, + assert!( + matches!( + sink.begin(&heap_meta(path)).await.unwrap(), + FileAction::Skip + ), "{path}" ); } - assert_eq!(sink.stats.files_seen, 0); + assert_eq!(ld(&sink.stats.files_seen), 0); } #[tokio::test] @@ -990,7 +1179,6 @@ mod tests { }) ); - let mut sink = sink; sink.start(&StartInfo { start_lsn: 0, timeline: 1, @@ -998,28 +1186,97 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(0); - let action = sink.begin(id, &meta).await.unwrap(); // Skip so source drains body without chunk() delivery; filtered // backfill pass relies on this for every non-opted rel - assert_eq!(action, FileAction::Skip); - sink.end(id).await.unwrap(); - assert!(sink.captured.is_empty()); - assert_eq!(sink.stats.files_seen, 1); - assert_eq!(sink.stats.files_skipped_unknown_filenode, 1); - assert_eq!(sink.stats.tuples_emitted, 0); - assert_eq!(sink.stats.pages_walked, 0); + assert!(matches!( + sink.begin(&heap_meta("base/5/16400")).await.unwrap(), + FileAction::Skip + )); + assert!(sink.captured().is_empty()); + assert_eq!(ld(&sink.stats.files_seen), 1); + assert_eq!(ld(&sink.stats.files_skipped_unknown_filenode), 1); + assert_eq!(ld(&sink.stats.tuples_emitted), 0); + assert_eq!(ld(&sink.stats.pages_walked), 0); + } + + /// Item 3.4: an unmapped relation's bytes drain off the wire without a + /// page decode. Before the filter the walk decoded every seeded rel and + /// the drain discarded them into `unsupported_relations`. + #[tokio::test] + async fn tap_filenode_filter_declines_unmapped_relations() { + let mut catalog = CatalogMap::new(); + let mapped = make_rel(); + let mut unmapped = make_rel(); + unmapped.rfn.rel_node = 16401; + unmapped.oid = 16401; + catalog.insert(Arc::new(mapped)); + catalog.insert(Arc::new(unmapped)); + let sink = PageWalkSink::new_capturing(catalog) + .with_tap_filenodes(Arc::new([(5, 16400)].into_iter().collect())); + sink.start(&StartInfo { + start_lsn: 0x1000, + timeline: 1, + tablespaces: Vec::new(), + }) + .await + .unwrap(); + + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; + entry.chunk(&synth_single_tuple_page(1)).await.unwrap(); + entry.end().await.unwrap(); + assert!(matches!( + sink.begin(&heap_meta("base/5/16401")).await.unwrap(), + FileAction::Skip + )); + + assert_eq!(sink.captured().len(), 1, "only the mapped rel decoded"); + assert_eq!(ld(&sink.stats.files_seen), 2); + assert_eq!(ld(&sink.stats.files_walked), 1); + assert_eq!(ld(&sink.stats.files_skipped_unmapped), 1); + assert_eq!(ld(&sink.stats.pages_walked), 1); + } + + /// A slab boundary must not lose the page it straddles, nor renumber + /// blocks: 64 KiB chunks against a 16-page slab put the split mid-page. + #[tokio::test] + async fn slab_boundary_keeps_every_page_and_block_number() { + let mut catalog = CatalogMap::new(); + catalog.insert(Arc::new(make_rel())); + let sink = PageWalkSink::new_capturing(catalog); + sink.start(&StartInfo { + start_lsn: 0x1000, + timeline: 1, + tablespaces: Vec::new(), + }) + .await + .unwrap(); + + // 40 pages, one tuple each, delivered in 6000-byte chunks so no + // chunk edge lands on a page or slab edge + let pages = 40usize; + let mut body = Vec::with_capacity(pages * PAGE_BYTES); + for i in 0..pages { + body.extend_from_slice(&synth_single_tuple_page(i as i32)); + } + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; + for chunk in body.chunks(6000) { + entry.chunk(chunk).await.unwrap(); + } + entry.end().await.unwrap(); + + let captured = sink.captured(); + assert_eq!(captured.len(), pages); + assert_eq!(ld(&sink.stats.pages_walked), pages as u64); + assert_eq!(ld(&sink.stats.tail_bytes_dropped), 0); + for (i, t) in captured.iter().enumerate() { + assert_eq!(t.blkno, i as u32, "page {i} kept its block number"); + assert!(matches!(t.columns[0], Some(ColumnValue::Int4(v)) if v == i as i32)); + } } #[tokio::test] async fn pagewalk_sink_rejects_non_base_paths() { - let mut sink = PageWalkSink::new_capturing(CatalogMap::new()); + let sink = PageWalkSink::new_capturing(CatalogMap::new()); sink.start(&StartInfo { start_lsn: 0, timeline: 1, @@ -1027,23 +1284,23 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("pg_control"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!( - sink.begin(EntryId(0), &meta).await.unwrap(), + assert!(matches!( + sink.begin(&FileMeta { + path: PathBuf::from("pg_control"), + size: 0, + mode: 0, + kind: FileKind::File, + }) + .await + .unwrap(), FileAction::Skip - ); + )); } /// object_store fan-out interleaves begin/chunk across concurrent - /// parts on the shared sink (mutex released across each body read). - /// Per-entry keying keeps each file's page-walk state separate; a - /// single `cur` slot would let a later begin clobber an in-flight - /// entry, misframing pages against the wrong relation. + /// parts. Each entry owns its walk state, so a later begin cannot + /// clobber an in-flight entry and misframe pages against the wrong + /// relation. #[tokio::test] async fn interleaved_entries_keep_independent_state() { let mut catalog = CatalogMap::new(); @@ -1053,7 +1310,7 @@ mod tests { rel_b.oid = 16401; catalog.insert(Arc::new(rel_a)); catalog.insert(Arc::new(rel_b)); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -1062,38 +1319,25 @@ mod tests { .await .unwrap(); - let meta_a = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let meta_b = FileMeta { - path: PathBuf::from("base/5/16401"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let (a, b) = (EntryId(0), EntryId(1)); - // Both open before either streams, chunks arrive reversed, ends // interleave: worst case for a shared slot - assert_eq!(sink.begin(a, &meta_a).await.unwrap(), FileAction::Tap); - assert_eq!(sink.begin(b, &meta_b).await.unwrap(), FileAction::Tap); - sink.chunk(b, &synth_single_tuple_page(200)).await.unwrap(); - sink.chunk(a, &synth_single_tuple_page(100)).await.unwrap(); - sink.end(a).await.unwrap(); - sink.end(b).await.unwrap(); + let mut a = tap(&sink, &heap_meta("base/5/16400")).await; + let mut b = tap(&sink, &heap_meta("base/5/16401")).await; + b.chunk(&synth_single_tuple_page(200)).await.unwrap(); + a.chunk(&synth_single_tuple_page(100)).await.unwrap(); + a.end().await.unwrap(); + b.end().await.unwrap(); // Each tuple carries its own file's rfn + value; a shared slot // would attribute both to rel B + let captured = sink.captured(); let mut by_rel: HashMap = HashMap::new(); - for t in &sink.captured { + for t in &captured { if let Some(Some(ColumnValue::Int4(v))) = t.columns.first() { by_rel.insert(t.rfn.rel_node, *v); } } - assert_eq!(sink.captured.len(), 2); + assert_eq!(captured.len(), 2); assert_eq!(by_rel.get(&16400), Some(&100), "entry A decoded vs rel A"); assert_eq!(by_rel.get(&16401), Some(&200), "entry B decoded vs rel B"); } diff --git a/src/backfill/backup_sink.rs b/src/backfill/backup_sink.rs index d1759200..6caa9f9a 100644 --- a/src/backfill/backup_sink.rs +++ b/src/backfill/backup_sink.rs @@ -11,15 +11,17 @@ //! `CatalogTracker::seed_from_source`. use std::io; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use async_trait::async_trait; use crate::backfill::backup_source::{ - BackupSink, EndInfo, EntryId, FileAction, FileKind, FileMeta, StartInfo, + BackupSink, EndInfo, FileAction, FileKind, FileMeta, StartInfo, }; use crate::backfill::pg_path::{is_system_dir, parse_base_path}; use crate::schema::FIRST_NORMAL_OBJECT_ID; -use ahash::{HashSet, HashSetExt}; +use ahash::HashSet; /// `relfilenode < 16384` is always bootstrap catalog. Rotated-catalog /// filenodes (`VACUUM FULL` / `REINDEX` on a catalog) land in `whitelist`. @@ -71,23 +73,24 @@ impl FromIterator<(u32, u32)> for CatalogFilenodes { /// (PG recovery refuses to start without them). pub struct DiskLanderSink { pub catalog_filenodes: CatalogFilenodes, - pub stats: DiskLanderStats, + pub stats: Arc, } -#[derive(Debug, Default, Clone)] -pub struct DiskLanderStats { - pub kept_files: u64, - pub kept_dirs: u64, - pub kept_symlinks: u64, - pub skipped_denylist: u64, - pub skipped_user_heap: u64, +crate::atomic_stats! { + pub struct DiskLanderStats { + pub kept_files, + pub kept_dirs, + pub kept_symlinks, + pub skipped_denylist, + pub skipped_user_heap, + } } impl DiskLanderSink { pub fn new(catalog_filenodes: CatalogFilenodes) -> Self { Self { catalog_filenodes, - stats: DiskLanderStats::default(), + stats: Arc::new(DiskLanderStats::default()), } } @@ -117,7 +120,7 @@ impl DiskLanderSink { } /// Distinguishes skip-denylist from skip-user-heap so MultiplexSink can -/// flip the latter to `Tap` when a page-walk sink is composed in. +/// flip the latter to a tap when a page-walk sink is composed in. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DiskAction { Keep, @@ -125,64 +128,56 @@ pub enum DiskAction { SkipUserHeap, } +impl DiskLanderSink { + /// Count one routing decision. Split out so `MultiplexSink` records the + /// lander's view of an entry it hands to the tap instead + fn record(&self, action: DiskAction, meta: &FileMeta) { + let bump = |c: &AtomicU64| { + c.fetch_add(1, Ordering::Relaxed); + }; + match action { + DiskAction::Keep => match meta.kind { + FileKind::File => bump(&self.stats.kept_files), + FileKind::Dir => bump(&self.stats.kept_dirs), + FileKind::Symlink { .. } => bump(&self.stats.kept_symlinks), + }, + DiskAction::SkipDenylist => bump(&self.stats.skipped_denylist), + DiskAction::SkipUserHeap => bump(&self.stats.skipped_user_heap), + } + } +} + #[async_trait] impl BackupSink for DiskLanderSink { - async fn begin(&mut self, _entry: EntryId, meta: &FileMeta) -> io::Result { - let action = match self.classify(meta) { + async fn begin(&self, meta: &FileMeta) -> io::Result { + let action = self.classify(meta); + self.record(action, meta); + Ok(match action { DiskAction::Keep => FileAction::Keep, DiskAction::SkipDenylist | DiskAction::SkipUserHeap => FileAction::Skip, - }; - if action == FileAction::Keep { - match meta.kind { - FileKind::File => self.stats.kept_files += 1, - FileKind::Dir => self.stats.kept_dirs += 1, - FileKind::Symlink { .. } => self.stats.kept_symlinks += 1, - } - } else { - match self.classify(meta) { - DiskAction::SkipDenylist => self.stats.skipped_denylist += 1, - DiskAction::SkipUserHeap => self.stats.skipped_user_heap += 1, - DiskAction::Keep => unreachable!(), - } - } - Ok(action) - } - async fn chunk(&mut self, _entry: EntryId, _bytes: &[u8]) -> io::Result<()> { - // Never returns Tap, so chunk() should never fire - Err(io::Error::other( - "DiskLanderSink::chunk called — sink only ever Keeps or Skips", - )) - } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - Ok(()) + }) } } -/// Multiplexes a DiskLanderSink (Keep / Skip) and a Tap-target sink -/// (typically PageWalkSink) over one source pass; begin() picks the -/// route, chunk/end follow it. +/// Routes a DiskLanderSink (Keep / Skip) and a tap-target sink +/// (typically PageWalkSink) over one source pass. Pure router: the tap's +/// own `EntrySink` carries the body, so nothing here is on the hot path. pub struct MultiplexSink { lander: DiskLanderSink, tap: T, - /// Entries routed to the tap. A set, not a flag, so concurrent - /// entries (object_store fan-out) dispatch to the right inner sink - /// instead of racing one shared bool. - tap_entries: HashSet, } impl MultiplexSink { pub fn new(lander: DiskLanderSink, tap: T) -> Self { - Self { - lander, - tap, - tap_entries: HashSet::new(), - } + Self { lander, tap } } - pub fn lander_stats(&self) -> &DiskLanderStats { + #[cfg(test)] + pub fn lander_stats(&self) -> &Arc { &self.lander.stats } + #[cfg(test)] pub fn into_inner(self) -> (DiskLanderSink, T) { (self.lander, self.tap) } @@ -190,49 +185,25 @@ impl MultiplexSink { #[async_trait] impl BackupSink for MultiplexSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { + async fn start(&self, info: &StartInfo) -> io::Result<()> { self.lander.start(info).await?; self.tap.start(info).await?; Ok(()) } - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result { - let action = match self.lander.classify(meta) { - DiskAction::Keep => { - self.lander.begin(entry, meta).await?; - FileAction::Keep - } - DiskAction::SkipDenylist => { - self.lander.begin(entry, meta).await?; - FileAction::Skip - } - DiskAction::SkipUserHeap => { - // Flip to Tap if the inner sink accepts; honour its - // decline (Skip / Keep) otherwise - let inner_action = self.tap.begin(entry, meta).await?; - if inner_action == FileAction::Tap { - self.tap_entries.insert(entry); - } - inner_action - } - }; - Ok(action) - } - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()> { - if self.tap_entries.contains(&entry) { - self.tap.chunk(entry, bytes).await - } else { - Err(io::Error::other("MultiplexSink: chunk without active tap")) + async fn begin(&self, meta: &FileMeta) -> io::Result { + let disk = self.lander.classify(meta); + if disk != DiskAction::SkipUserHeap { + return self.lander.begin(meta).await; } - } - async fn end(&mut self, entry: EntryId) -> io::Result<()> { - if self.tap_entries.remove(&entry) { - self.tap.end(entry).await?; - } else { - self.lander.end(entry).await?; + // Hand it to the tap; honour its decline (Skip / Keep) otherwise, + // counting the lander's own verdict only when it keeps the entry + let inner = self.tap.begin(meta).await?; + if !inner.is_tap() { + self.lander.record(disk, meta); } - Ok(()) + Ok(inner) } - async fn finish(&mut self, info: &EndInfo) -> io::Result<()> { + async fn finish(&self, info: &EndInfo) -> io::Result<()> { self.lander.finish(info).await?; self.tap.finish(info).await?; Ok(()) @@ -414,28 +385,43 @@ mod tests { } } - /// Counts begin / chunk / end, always returns Tap. Exercises - /// MultiplexSink without the full page walker. + fn ld(a: &AtomicU64) -> u64 { + a.load(Ordering::Relaxed) + } + + /// Counts begin / chunk / end, always taps. Exercises MultiplexSink + /// without the full page walker. #[derive(Debug, Default)] - struct CountingTap { - begins: u64, - chunks: u64, - ends: u64, - bytes: u64, + struct TapCounts { + begins: AtomicU64, + chunks: AtomicU64, + ends: AtomicU64, + bytes: AtomicU64, } + + struct CountingTap(Arc); + #[async_trait] impl BackupSink for CountingTap { - async fn begin(&mut self, _entry: EntryId, _meta: &FileMeta) -> io::Result { - self.begins += 1; - Ok(FileAction::Tap) + async fn begin(&self, _meta: &FileMeta) -> io::Result { + self.0.begins.fetch_add(1, Ordering::Relaxed); + Ok(FileAction::Tap(Box::new(CountingEntry(self.0.clone())))) } - async fn chunk(&mut self, _entry: EntryId, bytes: &[u8]) -> io::Result<()> { - self.chunks += 1; - self.bytes += bytes.len() as u64; + } + + struct CountingEntry(Arc); + + #[async_trait] + impl crate::backfill::backup_source::EntrySink for CountingEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.0.chunks.fetch_add(1, Ordering::Relaxed); + self.0 + .bytes + .fetch_add(bytes.len() as u64, Ordering::Relaxed); Ok(()) } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - self.ends += 1; + async fn end(self: Box) -> io::Result<()> { + self.0.ends.fetch_add(1, Ordering::Relaxed); Ok(()) } } @@ -443,49 +429,43 @@ mod tests { #[tokio::test] async fn multiplex_sink_routes_user_heap_to_tap() { let lander = DiskLanderSink::new(CatalogFilenodes::new()); - let tap = CountingTap::default(); - let mut mux = MultiplexSink::new(lander, tap); - - let m = FileMeta { - path: PathBuf::from("base/5/1259"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!(mux.begin(EntryId(0), &m).await.unwrap(), FileAction::Keep); - mux.end(EntryId(0)).await.unwrap(); + let counts = Arc::new(TapCounts::default()); + let mux = MultiplexSink::new(lander, CountingTap(counts.clone())); - let m = FileMeta { - path: PathBuf::from("base/5/16400"), + let file = |p: &str| FileMeta { + path: PathBuf::from(p), size: 0, mode: 0, kind: FileKind::File, }; - assert_eq!(mux.begin(EntryId(1), &m).await.unwrap(), FileAction::Tap); - mux.chunk(EntryId(1), &[0u8; 1024]).await.unwrap(); - mux.chunk(EntryId(1), &[1u8; 512]).await.unwrap(); - mux.end(EntryId(1)).await.unwrap(); + assert!(matches!( + mux.begin(&file("base/5/1259")).await.unwrap(), + FileAction::Keep + )); - let m = FileMeta { - path: PathBuf::from("pg_replslot/0/state"), - size: 0, - mode: 0, - kind: FileKind::File, + let FileAction::Tap(mut entry) = mux.begin(&file("base/5/16400")).await.unwrap() else { + panic!("user heap must tap"); }; - assert_eq!(mux.begin(EntryId(2), &m).await.unwrap(), FileAction::Skip); - mux.end(EntryId(2)).await.unwrap(); - - let (lander, tap) = mux.into_inner(); - assert_eq!(tap.begins, 1); - assert_eq!(tap.chunks, 2); - assert_eq!(tap.ends, 1); - assert_eq!(tap.bytes, 1536); - assert_eq!(lander.stats.kept_files, 1); - // User heap delegated to tap; lander never begin'd it so - // skipped_user_heap stays 0. tap.begins == 1 is the - // operator-visible "routed away from disk" signal. - assert_eq!(lander.stats.skipped_user_heap, 0); - assert_eq!(lander.stats.skipped_denylist, 1); + entry.chunk(&[0u8; 1024]).await.unwrap(); + entry.chunk(&[1u8; 512]).await.unwrap(); + entry.end().await.unwrap(); + + assert!(matches!( + mux.begin(&file("pg_replslot/0/state")).await.unwrap(), + FileAction::Skip + )); + + let (lander, _) = mux.into_inner(); + assert_eq!(ld(&counts.begins), 1); + assert_eq!(ld(&counts.chunks), 2); + assert_eq!(ld(&counts.ends), 1); + assert_eq!(ld(&counts.bytes), 1536); + assert_eq!(ld(&lander.stats.kept_files), 1); + // User heap delegated to tap, so the lander's own skip counter + // stays 0. `counts.begins == 1` is the operator-visible + // "routed away from disk" signal. + assert_eq!(ld(&lander.stats.skipped_user_heap), 0); + assert_eq!(ld(&lander.stats.skipped_denylist), 1); } #[test] @@ -505,26 +485,18 @@ mod tests { #[tokio::test] async fn multiplex_lander_stats_exposes_disk_counters() { let lander = DiskLanderSink::new(CatalogFilenodes::new()); - let mut mux = MultiplexSink::new(lander, CountingTap::default()); - let m = FileMeta { - path: PathBuf::from("base/5/1259"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!(mux.begin(EntryId(0), &m).await.unwrap(), FileAction::Keep); - mux.end(EntryId(0)).await.unwrap(); - let m = FileMeta { - path: PathBuf::from("pg_replslot/0/state"), + let mux = MultiplexSink::new(lander, CountingTap(Arc::default())); + let file = |p: &str| FileMeta { + path: PathBuf::from(p), size: 0, mode: 0, kind: FileKind::File, }; - assert_eq!(mux.begin(EntryId(1), &m).await.unwrap(), FileAction::Skip); - mux.end(EntryId(1)).await.unwrap(); + mux.begin(&file("base/5/1259")).await.unwrap(); + mux.begin(&file("pg_replslot/0/state")).await.unwrap(); let stats = mux.lander_stats(); - assert_eq!(stats.kept_files, 1); - assert_eq!(stats.skipped_denylist, 1); + assert_eq!(ld(&stats.kept_files), 1); + assert_eq!(ld(&stats.skipped_denylist), 1); } } diff --git a/src/backfill/backup_source.rs b/src/backfill/backup_source.rs index 7fc98f2d..1470a1df 100644 --- a/src/backfill/backup_source.rs +++ b/src/backfill/backup_source.rs @@ -25,14 +25,27 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::Ordering; +use std::time::Instant; use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncReadExt}; -use tokio::sync::Mutex; use walrus::pg::replication::base_backup::Tablespace; +crate::atomic_stats! { + /// Bootstrap pump stage attribution, shared from the orchestrator down + /// through the source. Measures what the pump hands over, not what a + /// sink made of it: entry counts live on the sinks' own stats + pub struct PumpStats { + /// Body bytes handed to a `Tap` entry + pub bytes_tapped, + /// Inside `EntrySink::chunk`: page framing, decode, channel send. + /// Against `page_walk.decode_nanos` this is the tap's overhead + pub sink_chunk_nanos, + } +} + /// Filesystem-object kind. Tar-driven sources translate tar entry types /// here; the trait does not expose tar. #[derive(Debug, Clone, PartialEq, Eq)] @@ -63,24 +76,48 @@ pub struct FileMeta { } /// Sink routing decision per file at `begin()`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FileAction { /// Source writes body / materializes dir or symlink under - /// `data_dir`; no `chunk()` + /// `data_dir`; no body callbacks Keep, - /// Source drains body unread; no land, no `chunk()` + /// Source drains body unread; no land, no body callbacks Skip, - /// Source streams body through `chunk()`, no land. Dir / Symlink - /// fire `chunk()` zero times - Tap, + /// Source streams the body into this owned sink, then drops it. + /// Owned rather than a shared-state decision so concurrent entries + /// need no lock on the body path. Dir / Symlink chunk zero times + Tap(Box), } -/// Per-file token threaded through `begin`/`chunk`/`end`. Sinks key -/// per-entry state on it so concurrent entries (object_store fan-out -/// under `buffer_unordered`, sink mutex released across body reads) -/// keep independent state. Monotonic per source run, globally unique. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct EntryId(pub u64); +impl FileAction { + pub fn is_tap(&self) -> bool { + matches!(self, FileAction::Tap(_)) + } +} + +impl std::fmt::Debug for FileAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + FileAction::Keep => "Keep", + FileAction::Skip => "Skip", + FileAction::Tap(_) => "Tap", + }) + } +} + +/// One `Tap` entry's body consumer. Owned by the source task driving that +/// entry, so nothing here contends with another entry: whatever state a +/// page walk or SLRU accumulator needs lives in the impl, and shared +/// state it does touch (catalog map, counters, channel sender) is +/// already `Sync`. +#[async_trait] +pub trait EntrySink: Send { + /// Body bytes in file order, arbitrary chunk sizes. Awaiting here + /// backpressures the source + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()>; + + /// Fires once, after the last `chunk` + async fn end(self: Box) -> io::Result<()>; +} /// Mirrors wal-rus `pg::replication::base_backup::StartInfo` so callers /// wired to wal-rus types don't translate. @@ -98,44 +135,54 @@ pub struct EndInfo { pub timeline: u32, } -/// Per-file consumer, driven by [`BackupSource::run`]: `start` once, -/// then per file `begin` / (if `Tap`) zero-or-more `chunk` / `end`, -/// then `finish` once. +/// Per-file router, driven by [`BackupSource::run`]: `start` once, then +/// `begin` per file, then `finish` once. A `Tap` decision hands back its +/// own [`EntrySink`], so the body path never re-enters here. /// -/// `Send` because parallel source impls drive the sink from worker -/// tasks. All methods async so a sink can backpressure its driver -/// directly: `chunk` awaiting a full downstream channel parks the body -/// read, which parks the source fetch. No intermediate buffer. +/// `&self` throughout: routing reads immutable state and bumps atomics, so +/// parallel source impls share one `Arc` with no lock. A source that holds +/// a lock across a body read serializes every other entry behind it, +/// which is what the owned entry sink exists to avoid. #[async_trait] -pub trait BackupSink: Send { - async fn start(&mut self, _info: &StartInfo) -> io::Result<()> { +pub trait BackupSink: Send + Sync { + async fn start(&self, _info: &StartInfo) -> io::Result<()> { Ok(()) } /// Must be cheap; per-file dispatch is the hot path - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result; + async fn begin(&self, meta: &FileMeta) -> io::Result; - /// Body bytes for a `Tap` entry, in file order per entry, arbitrary - /// chunk sizes. Calls for distinct entries may interleave. Awaiting - /// here backpressures the source. - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()>; + async fn finish(&self, _info: &EndInfo) -> io::Result<()> { + Ok(()) + } +} - /// Fires once per `begin()`, regardless of action returned - async fn end(&mut self, entry: EntryId) -> io::Result<()>; +/// Everything a tar-part driver needs behind one `Arc`, so a parallel +/// source hands its workers a clone instead of four arguments. `data_dir` +/// receives `Keep`d bodies. +pub struct PumpTarget { + pub data_dir: PathBuf, + pub sink: Arc, + pub stats: Arc, +} - async fn finish(&mut self, _info: &EndInfo) -> io::Result<()> { - Ok(()) +impl PumpTarget { + pub fn new(data_dir: PathBuf, sink: Arc, stats: Arc) -> Self { + Self { + data_dir, + sink, + stats, + } } } -/// `data_dir` receives `Keep`d bodies. Sink behind `Arc>` so -/// parallel source impls share it without per-worker copies. #[async_trait] pub trait BackupSource: Send { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> anyhow::Result<(StartInfo, EndInfo)>; } @@ -190,11 +237,9 @@ pub(crate) fn tar_entry_meta( /// Drain one tokio_tar archive against a sink, emitting per-entry /// callbacks. Called by `DirectSource` (per `BackupEvent::Archive.body`) /// and `ObjectStoreSource` (per fetched tar part). -pub(crate) async fn pump_tar_to_sink( +pub async fn pump_tar_to_sink( archive: &mut tokio_tar::Archive, - data_dir: &Path, - sink: &Arc>, - next_entry: &AtomicU64, + target: &PumpTarget, ) -> io::Result<()> where R: AsyncRead + Unpin + Send, @@ -207,35 +252,22 @@ where let Some(meta) = tar_entry_meta(&entry)? else { continue; }; - let id = EntryId(next_entry.fetch_add(1, Ordering::Relaxed)); - pump_entry(&mut entry, &meta, data_dir, sink, id).await?; + pump_entry(&mut entry, &meta, target).await?; } Ok(()) } /// One tar entry through the sink. Factored so callers can drive /// non-tar-shaped FileMeta sequences (e.g. inline symlink emission). -pub(crate) async fn pump_entry( - body: &mut R, - meta: &FileMeta, - data_dir: &Path, - sink: &Arc>, - entry: EntryId, -) -> io::Result<()> +pub async fn pump_entry(body: &mut R, meta: &FileMeta, target: &PumpTarget) -> io::Result<()> where R: AsyncRead + Unpin + ?Sized, { - let action = { - let mut s = sink.lock().await; - s.begin(entry, meta).await? - }; - match action { - FileAction::Keep => write_kept(body, meta, data_dir).await?, - FileAction::Skip => drain_to_void(body).await?, - FileAction::Tap => stream_to_sink(body, meta, sink, entry).await?, + match target.sink.begin(meta).await? { + FileAction::Keep => write_kept(body, meta, &target.data_dir).await, + FileAction::Skip => drain_to_void(body).await, + FileAction::Tap(entry) => stream_to_entry(body, meta, entry, &target.stats).await, } - let mut s = sink.lock().await; - s.end(entry).await } async fn drain_to_void(body: &mut R) -> io::Result<()> @@ -252,18 +284,18 @@ where Ok(()) } -async fn stream_to_sink( +async fn stream_to_entry( body: &mut R, meta: &FileMeta, - sink: &Arc>, - entry: EntryId, + mut entry: Box, + stats: &PumpStats, ) -> io::Result<()> where R: AsyncRead + Unpin + ?Sized, { if !matches!(meta.kind, FileKind::File) { drain_to_void(body).await?; - return Ok(()); + return entry.end().await; } let mut buf = [0u8; 64 * 1024]; loop { @@ -271,13 +303,16 @@ where if n == 0 { break; } - // Guard held across the chunk await: a full downstream channel - // parks this read (and any concurrent part contending the lock), - // propagating backpressure to the source fetch - let mut s = sink.lock().await; - s.chunk(entry, &buf[..n]).await?; + stats.bytes_tapped.fetch_add(n as u64, Ordering::Relaxed); + // Awaiting here parks only this entry's read, and through it this + // part's fetch. Other parts keep draining + let entered = Instant::now(); + entry.chunk(&buf[..n]).await?; + stats + .sink_chunk_nanos + .fetch_add(entered.elapsed().as_nanos() as u64, Ordering::Relaxed); } - Ok(()) + entry.end().await } async fn write_kept(body: &mut R, meta: &FileMeta, data_dir: &Path) -> io::Result<()> @@ -339,9 +374,7 @@ where #[allow(dead_code)] pub(crate) async fn emit_tablespace_symlink( tablespace: &Tablespace, - data_dir: &Path, - sink: &Arc>, - entry: EntryId, + target: &PumpTarget, ) -> io::Result<()> { if tablespace.is_default() { return Ok(()); @@ -355,7 +388,7 @@ pub(crate) async fn emit_tablespace_symlink( }, }; let mut body = tokio::io::empty(); - pump_entry(&mut body, &meta, data_dir, sink, entry).await + pump_entry(&mut body, &meta, target).await } #[cfg(test)] @@ -417,34 +450,59 @@ mod tests { use super::*; /// Collects every callback into `events` so tests assert on the - /// exact sequence; `tapped` holds captured Tap chunks per file. + /// exact sequence; `tapped` holds captured Tap bodies per file. #[derive(Debug, Default)] pub(crate) struct RecordingSink { - pub events: Vec, - pub tapped: Vec<(PathBuf, Vec)>, - cur_path: Option, - cur_tap: Option>, + pub events: std::sync::Mutex>, + pub tapped: std::sync::Mutex)>>, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Event { Start { start_lsn: u64, timeline: u32 }, - Begin { path: PathBuf, action: FileAction }, + Begin { path: PathBuf, action: &'static str }, Chunk { len: usize }, End { path: PathBuf }, Finish { end_lsn: u64 }, } + /// Owned per-entry recorder, so the sink itself stays lock-free on the + /// body path exactly as production sinks do + struct RecordingEntry { + sink: Arc, + path: PathBuf, + body: Vec, + } + + #[async_trait] + impl EntrySink for RecordingEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.sink + .events + .lock() + .unwrap() + .push(Event::Chunk { len: bytes.len() }); + self.body.extend_from_slice(bytes); + Ok(()) + } + async fn end(self: Box) -> io::Result<()> { + let RecordingEntry { sink, path, body } = *self; + sink.tapped.lock().unwrap().push((path.clone(), body)); + sink.events.lock().unwrap().push(Event::End { path }); + Ok(()) + } + } + #[async_trait] - impl BackupSink for RecordingSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { - self.events.push(Event::Start { + impl BackupSink for Arc { + async fn start(&self, info: &StartInfo) -> io::Result<()> { + self.events.lock().unwrap().push(Event::Start { start_lsn: info.start_lsn, timeline: info.timeline, }); Ok(()) } - async fn begin(&mut self, _entry: EntryId, meta: &FileMeta) -> io::Result { + async fn begin(&self, meta: &FileMeta) -> io::Result { let s = meta.path.to_string_lossy(); let action = if s.starts_with("pg_replslot/") { if matches!(meta.kind, FileKind::Dir) { @@ -453,35 +511,31 @@ mod tests { FileAction::Skip } } else if s == "base/5/16400" { - FileAction::Tap + FileAction::Tap(Box::new(RecordingEntry { + sink: self.clone(), + path: meta.path.clone(), + body: Vec::new(), + })) } else { FileAction::Keep }; - self.events.push(Event::Begin { + self.events.lock().unwrap().push(Event::Begin { path: meta.path.clone(), - action, + action: match action { + FileAction::Keep => "Keep", + FileAction::Skip => "Skip", + FileAction::Tap(_) => "Tap", + }, }); - self.cur_path = Some(meta.path.clone()); - self.cur_tap = (action == FileAction::Tap).then(Vec::new); - Ok(action) - } - async fn chunk(&mut self, _entry: EntryId, bytes: &[u8]) -> io::Result<()> { - self.events.push(Event::Chunk { len: bytes.len() }); - if let Some(buf) = self.cur_tap.as_mut() { - buf.extend_from_slice(bytes); - } - Ok(()) - } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - let path = self.cur_path.take().unwrap_or_default(); - if let Some(buf) = self.cur_tap.take() { - self.tapped.push((path.clone(), buf)); + if !action.is_tap() { + self.events.lock().unwrap().push(Event::End { + path: meta.path.clone(), + }); } - self.events.push(Event::End { path }); - Ok(()) + Ok(action) } - async fn finish(&mut self, info: &EndInfo) -> io::Result<()> { - self.events.push(Event::Finish { + async fn finish(&self, info: &EndInfo) -> io::Result<()> { + self.events.lock().unwrap().push(Event::Finish { end_lsn: info.end_lsn, }); Ok(()) @@ -573,13 +627,11 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let data_dir = tmp.path(); let tar_bytes = testing::build_synthetic_tar().await; - let recording = Arc::new(Mutex::new(RecordingSink::default())); - let sink: Arc> = recording.clone(); + let recording = Arc::new(RecordingSink::default()); + let sink: Arc = Arc::new(recording.clone()); let mut archive = tokio_tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let next_entry = AtomicU64::new(0); - pump_tar_to_sink(&mut archive, data_dir, &sink, &next_entry) - .await - .unwrap(); + let target = PumpTarget::new(data_dir.to_path_buf(), sink, Arc::new(PumpStats::default())); + pump_tar_to_sink(&mut archive, &target).await.unwrap(); assert!(data_dir.join("base/5/1259").exists(), "catalog must land"); assert!(data_dir.join("global/1213").exists(), "global must land"); @@ -590,10 +642,9 @@ mod tests { // User heap tapped, not landed assert!(!data_dir.join("base/5/16400").exists()); - let r = recording.lock().await; + let events = recording.events.lock().unwrap(); // Last file event must be pg_control end (contract 3) - let last_end = r - .events + let last_end = events .iter() .rev() .find_map(|e| match e { @@ -603,8 +654,7 @@ mod tests { .unwrap(); assert_eq!(last_end, PathBuf::from("pg_control")); // Tapped chunks sum to the file body length - let tapped_bytes: usize = r - .events + let tapped_bytes: usize = events .iter() .filter_map(|e| match e { Event::Chunk { len } => Some(*len), diff --git a/src/backfill/backup_source_direct.rs b/src/backfill/backup_source_direct.rs index c0d234c6..204a5ede 100644 --- a/src/backfill/backup_source_direct.rs +++ b/src/backfill/backup_source_direct.rs @@ -10,18 +10,17 @@ use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicU64; use anyhow::{Context, Result, bail}; use async_trait::async_trait; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc; use walrus::pg::replication::base_backup::{ BackupEvent, BaseBackupOpts, ChannelReader, run_base_backup, }; use walrus::pg::replication::conn::{PgConfig, ReplicationConn}; use crate::backfill::backup_source::{ - BackupSink, BackupSource, EndInfo, StartInfo, pump_tar_to_sink, + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, }; /// Replication-protocol BASE_BACKUP issuer @@ -41,7 +40,8 @@ impl BackupSource for DirectSource { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { let DirectSource { source, opts } = *self; @@ -58,9 +58,7 @@ impl BackupSource for DirectSource { let mut start: Option = None; let mut end: Option = None; - // Archives drain sequentially here; shared counter kept for - // symmetry with object_store's concurrent parts - let next_entry = AtomicU64::new(0); + let target = PumpTarget::new(data_dir, sink.clone(), stats); while let Some(ev) = rx.recv().await { let ev = ev.context("DirectSource: BASE_BACKUP event channel")?; @@ -71,10 +69,7 @@ impl BackupSource for DirectSource { timeline: s.timeline, tablespaces: s.tablespaces, }; - { - let mut g = sink.lock().await; - g.start(&s).await?; - } + sink.start(&s).await?; start = Some(s); } BackupEvent::Archive { meta, body } => { @@ -84,17 +79,14 @@ impl BackupSource for DirectSource { oid = meta.oid, "archive open", ); - drive_archive(body, &data_dir, sink.clone(), &next_entry).await?; + drive_archive(body, &target).await?; } BackupEvent::Finish(e) => { let e = EndInfo { end_lsn: e.end_lsn, timeline: e.timeline, }; - { - let mut g = sink.lock().await; - g.finish(&e).await?; - } + sink.finish(&e).await?; end = Some(e); } } @@ -115,13 +107,11 @@ impl BackupSource for DirectSource { /// SyncIoBridge / spawn_blocking. async fn drive_archive( body: mpsc::Receiver>, - data_dir: &std::path::Path, - sink: Arc>, - next_entry: &AtomicU64, + target: &PumpTarget, ) -> Result<()> { let reader = ChannelReader::new(body); let mut archive = tokio_tar::Archive::new(reader); - pump_tar_to_sink(&mut archive, data_dir, &sink, next_entry) + pump_tar_to_sink(&mut archive, target) .await .context("DirectSource: tar unpack")?; Ok(()) diff --git a/src/backfill/backup_source_object_store.rs b/src/backfill/backup_source_object_store.rs index 1a9d557c..3f6a890a 100644 --- a/src/backfill/backup_source_object_store.rs +++ b/src/backfill/backup_source_object_store.rs @@ -28,9 +28,6 @@ use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicU64; - -use tokio::sync::Mutex; use anyhow::{Context, Result, bail}; use async_trait::async_trait; @@ -45,7 +42,7 @@ use crate::backfill::backup_sentinel::build_lsn_pair; #[cfg(test)] use crate::backfill::backup_sentinel::{parse_timeline_from_name, tablespaces_from_spec}; use crate::backfill::backup_source::{ - BackupSink, BackupSource, EndInfo, StartInfo, pump_tar_to_sink, + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, }; /// `parallelism` bounds in-flight data parts; `pg_control` always runs @@ -90,7 +87,8 @@ impl BackupSource for ObjectStoreSource { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { let ObjectStoreSource { settings, @@ -124,10 +122,7 @@ impl BackupSource for ObjectStoreSource { } let (start, end) = build_lsn_pair(&resolved, &sentinel)?; - { - let mut g = sink.lock().await; - g.start(&start).await?; - } + sink.start(&start).await?; let parts = list_tar_parts(&storage, &resolved).await?; if parts.is_empty() { @@ -149,10 +144,7 @@ impl BackupSource for ObjectStoreSource { "draining tar partitions" ); - // Shared counter across concurrent parts; unique EntryId per - // entry keeps interleaved begin/chunk on the shared sink mutex - // from clobbering each other's page-walk slot. - let next_entry = Arc::new(AtomicU64::new(0)); + let target = Arc::new(PumpTarget::new(data_dir, sink.clone(), stats)); // Phase A: bounded fan-out of data parts via buffer_unordered // try_collect short-circuits on the first part error. A plain @@ -161,16 +153,9 @@ impl BackupSource for ObjectStoreSource { .map(|key| { let storage = storage.clone(); let settings = settings.clone(); - let data_dir = data_dir.clone(); - let sink = sink.clone(); - let next_entry = next_entry.clone(); let spool = part_spool_dir.clone(); - async move { - unpack_one_part( - &settings, &storage, &key, &data_dir, &spool, sink, next_entry, - ) - .await - } + let target = target.clone(); + async move { unpack_one_part(&settings, &storage, &key, &spool, &target).await } }) .buffer_unordered(parallelism) .try_collect::>() @@ -179,22 +164,10 @@ impl BackupSource for ObjectStoreSource { // Phase B: pg_control barrier, single-task. wal-g emits one // control part; walk in sorted order if ever more for key in &control_parts { - unpack_one_part( - &settings, - &storage, - key, - &data_dir, - &part_spool_dir, - sink.clone(), - next_entry.clone(), - ) - .await?; + unpack_one_part(&settings, &storage, key, &part_spool_dir, &target).await?; } - { - let mut g = sink.lock().await; - g.finish(&end).await?; - } + sink.finish(&end).await?; Ok((start, end)) } } @@ -206,10 +179,8 @@ async fn unpack_one_part( settings: &Settings, storage: &DynStorage, key: &str, - data_dir: &std::path::Path, part_spool_dir: &std::path::Path, - sink: Arc>, - next_entry: Arc, + target: &PumpTarget, ) -> Result<()> { let method = method_from_key(key); let body = storage @@ -222,7 +193,7 @@ async fn unpack_one_part( let decoded = compression::decode(method, Box::pin(spooled)); let mut archive = tokio_tar::Archive::new(decoded); - pump_tar_to_sink(&mut archive, data_dir, &sink, &next_entry) + pump_tar_to_sink(&mut archive, target) .await .with_context(|| format!("ObjectStoreSource: tar unpack {key}"))?; tracing::info!( @@ -242,6 +213,13 @@ async fn unpack_one_part( /// so no error or panic path can leak it. Same trick as /// [`crate::spill::BodySpoolFile`]. Bytes are still compressed here, so /// scratch tracks object size rather than the unpacked tar. +/// +/// Per-entry tap sinks removed the shared-lock stall this was also +/// covering, and the page walk outruns any realistic GET, but the spool +/// stays: the pipeline it feeds backpressures +/// on ClickHouse by design, so a live body would be held for however long +/// CH is the limiter, which the 60s cap does not allow. Decode speed was +/// never the binding term. async fn spool_backup_part( dir: &std::path::Path, key: &str, diff --git a/src/backfill/bootstrap_oracle.rs b/src/backfill/bootstrap_oracle.rs index d4eceab8..bd85db26 100644 --- a/src/backfill/bootstrap_oracle.rs +++ b/src/backfill/bootstrap_oracle.rs @@ -13,6 +13,7 @@ use crate::column_rules::ColumnRules; use crate::emit::ch_emitter::TablePlan; use crate::mapping::{MappingSnapshot, SystemColumns}; use crate::ops::oracle::Oracle; +use crate::schema::RelName; const ORACLE_PORT: u16 = 55440; @@ -28,6 +29,7 @@ impl BootstrapOracle { source_conninfo: String, source_password: Option, bridge_lib_dir: Option, + workers: usize, connect_budget: Duration, ) -> Result { let data_dir = base_dir.join("pg"); @@ -57,6 +59,7 @@ impl BootstrapOracle { let mut bridge = BridgeConf::in_dir(&b_sock); bridge.socket_path = b_bridge; bridge.library_dir = bridge_lib_dir; + bridge.workers = workers; let cfg_b = oracle_cfg(&b_data, &b_base, &b_sock, Some(bridge)); let b = Shadow::new(cfg_b); b.write_base_conf().context("serve conf")?; @@ -67,9 +70,10 @@ impl BootstrapOracle { .context("bootstrap oracle provision task")? .context("bootstrap oracle provision")?; - let bridge = crate::ops::bridge::connect_with_budget(&bridge_socket, connect_budget) - .await - .context("bootstrap oracle bridge connect")?; + let bridge = + crate::ops::bridge::connect_with_budget(&bridge_socket, workers, connect_budget) + .await + .context("bootstrap oracle bridge connect")?; Ok(Self { shadow, oracle: Arc::new(Oracle::new(Arc::new(bridge))), @@ -124,19 +128,28 @@ fn run_pg_dump(conninfo: &str, password: Option<&str>) -> Result { String::from_utf8(out.stdout).context("pg_dump output not utf8") } +/// Provisioning costs an `initdb` + `pg_dump` + apply + restart, so gate it +/// on the relations that actually reach ClickHouse. `walked` narrows the +/// mapped set further to what the greenfield snapshot page-walks: an +/// `initial_load = "none"` relation ships no bootstrap row, so its oracle +/// columns are not a reason to stand up a side Postgres pub fn needs_oracle( catalog: &CatalogMap, tables: &MappingSnapshot, column_rules: &ColumnRules, + walked: impl Fn(&RelName) -> bool, ) -> bool { let alloc = Allocator::stdlib(); let system = SystemColumns::default(); - catalog.descriptors().any(|desc| { - tables.get(&desc.rel_name).is_some_and(|mapping| { - TablePlan::build(alloc, desc, mapping, column_rules, &system) - .map_or(true, |plan| plan.needs_oracle()) + catalog + .descriptors() + .filter(|d| walked(&d.rel_name)) + .any(|desc| { + tables.get(&desc.rel_name).is_some_and(|mapping| { + TablePlan::build(alloc, desc, mapping, column_rules, &system) + .map_or(true, |plan| plan.needs_oracle()) + }) }) - }) } #[cfg(test)] @@ -230,7 +243,7 @@ mod tests { "Nullable(JSON)", "premise: default bridge maps json to a composite CH target", ); - assert!(needs_oracle(&catalog, &tables, &rules)); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] @@ -243,7 +256,7 @@ mod tests { ]), &rules, ); - assert!(!needs_oracle(&catalog, &tables, &rules)); + assert!(!needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] @@ -256,13 +269,27 @@ mod tests { ]), &rules, ); - assert!(needs_oracle(&catalog, &tables, &rules)); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] fn unmapped_relation_needs_no_oracle() { let rules = ColumnRules::default(); let (catalog, _) = bridged(rel(vec![attr(1, "doc", JSONOID, "json", -1)]), &rules); - assert!(!needs_oracle(&catalog, &Arc::default(), &rules)); + assert!(!needs_oracle(&catalog, &Arc::default(), &rules, |_| true)); + } + + #[test] + fn relation_out_of_the_snapshot_needs_no_oracle() { + let rules = ColumnRules::default(); + let (catalog, tables) = bridged( + rel(vec![ + attr(1, "id", INT4OID, "int4", 4), + attr(2, "doc", JSONOID, "json", -1), + ]), + &rules, + ); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); + assert!(!needs_oracle(&catalog, &tables, &rules, |_| false)); } } diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 1aafc293..69871dea 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -84,6 +84,13 @@ use crate::schema::{ }; use crate::source::source_feed::open_sql_client; use crate::toast::ToastResolver; + +/// Rows per COPY-backfill channel hop. Byte trigger below bounds the wide-row +/// case, so this only caps the narrow-row hop rate +const COPY_SLAB_ROWS: usize = 1024; +/// Decoded value bytes per hop. With +/// [`BOOTSTRAP_TUPLE_CHANNEL_CAP`] this is the resident payload ceiling +const COPY_SLAB_BYTES: usize = 1 << 20; use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; const LEDGER_FILENAME: &str = "backfills.toml"; @@ -1071,7 +1078,7 @@ impl CopyBackfiller { let mut catalog = CatalogMap::new(); catalog.insert(desc.clone()); - let (tup_tx, tup_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (tup_tx, tup_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); let drain = tokio::spawn(bootstrap::drain( tup_rx, catalog, @@ -1101,6 +1108,11 @@ impl CopyBackfiller { let stream = BinaryCopyOutStream::new(copy, &byte_fields); futures::pin_mut!(stream); let mut rows = 0u64; + // Slab the channel: one hop per `COPY_SLAB_BYTES` of decoded + // values, not one per row. Byte-triggered so wide rows keep the + // resident bound whatever the row count + let mut slab: Vec = Vec::new(); + let mut slab_bytes = 0usize; while let Some(row) = stream.next().await { let row = row.context("backfill: COPY stream")?; let mut columns: Vec> = vec![None; plan.natts]; @@ -1113,22 +1125,37 @@ impl CopyBackfiller { .unwrap_or(ColumnValue::Null); columns[(cp.attnum - 1).max(0) as usize] = Some(v); } + slab_bytes += columns + .iter() + .flatten() + .map(ColumnValue::approx_bytes) + .sum::(); + slab.push(BackfillTuple { + rfn: desc.rfn, + xid: 0, + xmax: 0, + infomask: 0, + source_lsn: s_lsn.get(), + // COPY text rows have no on-page TID (values arrive + // detoasted, no chunks flow here) + blkno: 0, + offnum: 0, + columns, + }); + rows += 1; + if slab.len() >= COPY_SLAB_ROWS || slab_bytes >= COPY_SLAB_BYTES { + slab_bytes = 0; + tup_tx + .send(std::mem::take(&mut slab)) + .await + .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; + } + } + if !slab.is_empty() { tup_tx - .send(BackfillTuple { - rfn: desc.rfn, - xid: 0, - xmax: 0, - infomask: 0, - source_lsn: s_lsn.get(), - // COPY text rows have no on-page TID (values arrive - // detoasted, no chunks flow here) - blkno: 0, - offnum: 0, - columns, - }) + .send(slab) .await .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; - rows += 1; } rows }; diff --git a/src/bin/stream.rs b/src/bin/stream.rs index b4b750d2..19d2fddf 100644 --- a/src/bin/stream.rs +++ b/src/bin/stream.rs @@ -49,7 +49,8 @@ use walrus::pg::replication::base_backup::BaseBackupOpts; use walrus::pg::replication::conn::PgConfig; use walrus::pg::replication::tls::SslMode; use walshadow::backfill_bootstrap::{ - BootstrapConfig, BootstrapOutcome, drain_backfill, seed_in_snapshot, spawn_greenfield_bootstrap, + BootstrapConfig, BootstrapOutcome, BootstrapProgress, drain_backfill, seed_in_snapshot, + spawn_greenfield_bootstrap, }; use walshadow::backup_source::BackupSource; use walshadow::backup_source_direct::DirectSource; @@ -385,6 +386,12 @@ struct Args { /// `dynamic_library_path`. #[arg(long)] bridge_lib_dir: Option, + /// Bridge workers to run on a daemon-owned shadow, and sockets to pool + /// against. Each worker serves one request at a time, so a value below + /// the decode pool's width caps oracle throughput at that many + /// concurrent round trips whatever the daemon does. + #[arg(long, default_value_t = 1)] + bridge_workers: usize, /// Walsender bind address. `127.0.0.1:0` lets the kernel pick a free /// port, valid only for externally managed shadow (no /// `--bootstrap-shadow-data-dir`): operator reads @@ -937,6 +944,7 @@ async fn run_session( .with_context(|| format!("ensure physical replication slot {slot}"))?; tracing::info!(target: "walshadow", slot, "physical replication slot ready"); } + let mut bootstrap_progress: Option = None; let bootstrap_end_lsn: Option = if matches!(shadow_start, ShadowStart::Bootstrap(_)) { if !args.skip_preflight { let source_sql = feed @@ -952,11 +960,18 @@ async fn run_session( .into_result() .context("pre-flight rejected bootstrap")?; } - Some( - run_bootstrap(&cfg, &mut feed, args, &bootstrap_plan, ch_config.clone()) - .await - .context("bootstrap")?, + let (end_lsn, progress) = run_bootstrap( + &cfg, + &mut feed, + args, + &bootstrap_plan, + ch_config.clone(), + &metrics, ) + .await + .context("bootstrap")?; + bootstrap_progress = Some(progress); + Some(end_lsn) } else { None }; @@ -1242,7 +1257,7 @@ async fn run_session( let connect_budget = Duration::from_secs(args.shadow_connect_timeout); let bridge_path = args.bridge_socket_path(); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(&bridge_path, connect_budget) + walshadow::bridge::connect_with_budget(&bridge_path, args.bridge_workers, connect_budget) .await .with_context(|| format!("connect bridge at {}", bridge_path.display()))?, ); @@ -1250,6 +1265,7 @@ async fn run_session( tracing::info!( target: "walshadow::bridge", socket = %bridge_path.display(), + workers = bridge.pool_size(), pg_version = info.map(|i| i.pg_version_num).unwrap_or(0), in_recovery = info.map(|i| i.in_recovery).unwrap_or(false), "bridge connected", @@ -1724,6 +1740,7 @@ async fn run_session( addr = %addr, decoders, inserters, + resolvers = bridge.pool_size(), "parallel decode+insert pipeline starting", ); PipelineConfig { @@ -2667,6 +2684,7 @@ async fn run_session( &desc_log, metrics_resolver.as_deref(), metrics_backfiller.as_deref(), + bootstrap_progress.as_ref(), ) .await; if advanced { @@ -3313,6 +3331,7 @@ async fn populate_metrics( desc_log: &walshadow::desc_log::DescriptorLog, config_resolver: Option<&ConfigResolver>, backfiller: Option<&walshadow::copy_backfill::CopyBackfiller>, + bootstrap: Option<&BootstrapProgress>, ) { use std::collections::BTreeMap; use walshadow::record::rmgr_label; @@ -3482,6 +3501,15 @@ async fn populate_metrics( inserter_batches_in_total: emitter_stats .map(|s| s.inserter_batches_in.load(Ordering::Relaxed)) .unwrap_or(0), + inserter_ch_seconds_total: emitter_stats + .map(|s| s.inserter_ch_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), + inserter_encode_seconds_total: emitter_stats + .map(|s| s.inserter_encode_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), + oracle_resolve_seconds_total: emitter_stats + .map(|s| s.oracle_resolve_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), process_cpu_seconds_total: proc_cpu, process_resident_memory_bytes: proc_rss, emitter_xacts_total: emitter_stats @@ -3493,6 +3521,9 @@ async fn populate_metrics( emitter_deletes_discarded: emitter_stats .map(|s| s.deletes_discarded.load(Ordering::Relaxed)) .unwrap_or(0), + oracle_local_columns_total: emitter_stats + .map(|s| s.oracle_local_columns.load(Ordering::Relaxed)) + .unwrap_or(0), oracle_blocks_total: oracle_stats .map(|s| s.blocks.load(Ordering::Relaxed)) .unwrap_or(0), @@ -3585,6 +3616,10 @@ async fn populate_metrics( bridge_requests_by_op: bridge_ops(bridge_stats.map(|b| &b.requests)), bridge_errors_by_op: bridge_ops(bridge_stats.map(|b| &b.errors)), bridge_request_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.request_nanos)), + bridge_lock_wait_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.lock_wait_nanos)), + bridge_service_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.service_nanos)), + bridge_request_bytes_by_op: bridge_ops(bridge_stats.map(|b| &b.request_bytes)), + bridge_response_bytes_by_op: bridge_ops(bridge_stats.map(|b| &b.response_bytes)), bridge_reconnects_total: bridge_gauge(bridge_stats, |b| &b.reconnects), bridge_scan_rows_total: bridge_gauge(bridge_stats, |b| &b.scan_rows), bridge_scan_replay_moved_total: bridge_gauge(bridge_stats, |b| &b.scan_replay_moved), @@ -3592,10 +3627,32 @@ async fn populate_metrics( &b.scan_subtrans_mismatch }), bridge_native_bytes_total: bridge_gauge(bridge_stats, |b| &b.native_bytes), + ..bootstrap_gauges(bootstrap) }; registry.set(snap).await; } +/// Bootstrap stage attribution, frozen at its final values once the pump +/// returns. Rendered for the whole session so a slow initial load stays +/// attributable after the fact +fn bootstrap_gauges(progress: Option<&BootstrapProgress>) -> MetricsSnapshot { + let Some(p) = progress else { + return MetricsSnapshot::default(); + }; + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed); + MetricsSnapshot { + bootstrap_bytes_tapped: ld(&p.pump.bytes_tapped), + bootstrap_pages_walked: ld(&p.page_walk.pages_walked), + bootstrap_tuples_emitted: ld(&p.page_walk.tuples_emitted), + bootstrap_files_walked: ld(&p.page_walk.files_walked), + bootstrap_files_skipped_unmapped: ld(&p.page_walk.files_skipped_unmapped), + bootstrap_decode_seconds: ld(&p.page_walk.decode_nanos) as f64 / 1e9, + bootstrap_tap_seconds: ld(&p.pump.sink_chunk_nanos) as f64 / 1e9, + bootstrap_channel_block_seconds: ld(&p.page_walk.channel_block_nanos) as f64 / 1e9, + ..MetricsSnapshot::default() + } +} + /// Zero when bridge stats are unavailable, so series stays present fn bridge_gauge( stats: Option<&walshadow::bridge::BridgeStats>, @@ -4168,7 +4225,9 @@ async fn run_bootstrap( args: &Args, plan: &BootstrapPlan, ch_config: Option, -) -> Result { + metrics: &MetricsRegistry, +) -> Result<(u64, BootstrapProgress)> { + let bootstrap_started = Instant::now(); let shadow_data_dir = args .bootstrap_shadow_data_dir .clone() @@ -4284,19 +4343,114 @@ async fn run_bootstrap( let (mapping, resolved) = bootstrap_build_mapping(&emitter_cfg, &drain_catalog, args) .await .context("bootstrap: build mapping")?; - Some((emitter_cfg, mapping, resolved)) + // `initial_load = "none"` (table override, else namespace) opts a + // relation out of the greenfield snapshot: create it + stream CDC, + // but don't page-walk its existing rows. + let skip_initial: std::collections::HashSet<_> = drain_catalog + .descriptors() + .filter_map(|d| { + let rn = &d.rel_name; + let none = match emitter_cfg.table_initial_loads.get(rn) { + Some(s) => s.parse::() == Ok(InitialLoadMode::None), + None => { + resolved + .namespaces + .get(rn.namespace.as_ref()) + .and_then(|n| n.initial_load) + == Some(InitialLoadMode::None) + } + }; + none.then(|| rn.clone()) + }) + .collect(); + Some((emitter_cfg, mapping, resolved, skip_initial)) } None => None, }; + // Decline unmapped relations at `begin` so their pages never decode. + // Metrics-only (no CH) has no mapping to filter against, so it walks all + let (tap_filenodes, needs_oracle) = match &ch_target { + Some((_, mapping, resolved, skip_initial)) => { + let mapped = mapping.snapshot().await; + let walked = |rn: &RelName| mapped.contains_key(rn) && !skip_initial.contains(rn); + ( + walshadow::backfill_bootstrap::tap_filenode_set(&drain_catalog, walked) + .map(Arc::new), + walshadow::backfill::bootstrap_oracle::needs_oracle( + &drain_catalog, + &mapped, + &resolved.column_rules, + walked, + ), + ) + } + None => (None, false), + }; + + // Off the backup window: provisioning is an initdb + pg_dump + apply + + // restart, and doing it after `BASE_BACKUP` opens parks a live backup + // through all of it — in object-store mode against walrus's 60 s request + // cap + let bootstrap_oracle = if needs_oracle { + let source_conninfo = format!( + "host={} port={} user={} dbname={} sslmode={}", + src_cfg.host, + src_cfg.port, + src_cfg.user, + src_cfg.database, + if src_cfg.sslmode == SslMode::Disable { + "disable" + } else { + "prefer" + }, + ); + Some( + walshadow::backfill::bootstrap_oracle::BootstrapOracle::provision( + args.spill_dir.join("bootstrap_oracle"), + source_conninfo, + src_cfg.password.clone(), + args.bridge_lib_dir.clone(), + args.bridge_workers, + Duration::from_secs(args.shadow_connect_timeout), + ) + .await + .context( + "bootstrap oracle: greenfield needs it to resolve tier-3 types; \ + refusing to load empty columns", + )?, + ) + } else { + None + }; + let oracle = bootstrap_oracle.as_ref().map(|o| o.oracle()); + prepare_bootstrap_dir(&shadow_data_dir) .await .context("prepare shadow data dir for bootstrap")?; - let cfg = BootstrapConfig::new(shadow_data_dir.clone()); + let mut cfg = BootstrapConfig::new(shadow_data_dir.clone()); + if let Some(set) = tap_filenodes { + cfg = cfg.with_tap_filenodes(set); + } + let progress = cfg.progress.clone(); + // Only writer of the registry until the status loop starts, so a plain + // gauge-only snapshot is the whole surface here + let ticker = tokio::spawn({ + let metrics = metrics.clone(); + let progress = progress.clone(); + async move { + let mut tick = tokio::time::interval(Duration::from_secs(5)); + loop { + tick.tick().await; + metrics.set(bootstrap_gauges(Some(&progress))).await; + } + } + }); let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); - let (shipped, outcome) = if let Some((emitter_cfg, mapping, resolved)) = ch_target { + let (shipped, outcome) = if let Some((emitter_cfg, mapping, resolved, skip_initial)) = ch_target + { // Route bootstrap rows through the shared insert tail. Bootstrap // is the easy case: every row op=Insert at _lsn = start_lsn, no // aborts / TRUNCATE / DDL. Keep operator's flush_timeout; tail @@ -4310,41 +4464,6 @@ async fn run_bootstrap( let fatal = Fatal::new(); let inserter_pool_size = emitter_cfg.inserter_pool_size; - let source_conninfo = format!( - "host={} port={} user={} dbname={} sslmode={}", - src_cfg.host, - src_cfg.port, - src_cfg.user, - src_cfg.database, - if src_cfg.sslmode == SslMode::Disable { - "disable" - } else { - "prefer" - }, - ); - let bootstrap_oracle = if walshadow::backfill::bootstrap_oracle::needs_oracle( - &drain_catalog, - &mapping.snapshot().await, - &resolved.column_rules, - ) { - Some( - walshadow::backfill::bootstrap_oracle::BootstrapOracle::provision( - args.spill_dir.join("bootstrap_oracle"), - source_conninfo, - src_cfg.password.clone(), - args.bridge_lib_dir.clone(), - Duration::from_secs(args.shadow_connect_timeout), - ) - .await - .context( - "bootstrap oracle: greenfield needs it to convert oracle columns; \ - refusing to load empty columns", - )?, - ) - } else { - None - }; - let (msg_tx, ack, tail) = tail::spawn_with_config( &emitter_cfg, inserter_pool_size, @@ -4352,7 +4471,7 @@ async fn run_bootstrap( emitter_ack, fatal.clone(), None, - bootstrap_oracle.as_ref().map(|o| o.oracle()), + oracle, ) .await .context("bootstrap: spawn insert tail")?; @@ -4363,27 +4482,6 @@ async fn run_bootstrap( "bootstrap insert tail started", ); - // `initial_load = "none"` (table override, else namespace) opts a - // relation out of the greenfield snapshot: create it + stream CDC, but - // don't page-walk its existing rows. - let skip_initial: std::collections::HashSet<_> = drain_catalog - .descriptors() - .filter_map(|d| { - let rn = &d.rel_name; - let none = match emitter_cfg.table_initial_loads.get(rn) { - Some(s) => s.parse::() == Ok(InitialLoadMode::None), - None => { - resolved - .namespaces - .get(rn.namespace.as_ref()) - .and_then(|n| n.initial_load) - == Some(InitialLoadMode::None) - } - }; - none.then(|| rn.clone()) - }) - .collect(); - let deferred_path = args.spill_dir.join("bootstrap_deferred.bin"); tokio::fs::remove_file(&deferred_path).await.ok(); let drain = tokio::spawn(bootstrap::drain( @@ -4437,18 +4535,33 @@ async fn run_bootstrap( (shipped, outcome) }; + let ld = |a: &std::sync::atomic::AtomicU64| a.load(Ordering::Relaxed); tracing::info!( target: "walshadow::bootstrap", start_lsn = format_pg_lsn(outcome.start.start_lsn).to_string(), end_lsn = format_pg_lsn(outcome.end.end_lsn).to_string(), timeline = outcome.start.timeline, - kept_files = outcome.disk.kept_files, - skipped_denylist = outcome.disk.skipped_denylist, - files_walked = outcome.page_walk.files_walked, - tuples_emitted = outcome.page_walk.tuples_emitted, + kept_files = ld(&outcome.disk.kept_files), + skipped_denylist = ld(&outcome.disk.skipped_denylist), + files_walked = ld(&outcome.page_walk.files_walked), + tuples_emitted = ld(&outcome.page_walk.tuples_emitted), drained = shipped, "bootstrap landed", ); + // Stage attribution: which of tap, decode or emitter drain owned the + // wall clock. Sum exceeds elapsed under source parallelism + tracing::info!( + target: "walshadow::bootstrap", + elapsed_secs = bootstrap_started.elapsed().as_secs_f64(), + bytes_tapped = ld(&outcome.pump.bytes_tapped), + pages_walked = ld(&outcome.page_walk.pages_walked), + tap_secs = ld(&outcome.pump.sink_chunk_nanos) as f64 / 1e9, + decode_secs = ld(&outcome.page_walk.decode_nanos) as f64 / 1e9, + channel_block_secs = ld(&outcome.page_walk.channel_block_nanos) as f64 / 1e9, + files_skipped_unmapped = ld(&outcome.page_walk.files_skipped_unmapped), + "bootstrap stage timings", + ); + ticker.abort(); if let Some((settings, storage)) = wal_hydrate { fetch_wal_into_pg_wal( @@ -4479,7 +4592,7 @@ async fn run_bootstrap( .await .context("clear completed bootstrap marker")?; - Ok(outcome.end.end_lsn) + Ok((outcome.end.end_lsn, progress)) } /// Routing map for the bootstrap drain: explicit `[table.*]` seeded up front, @@ -4639,6 +4752,7 @@ fn build_owned_shadow(args: &Args, data_dir: PathBuf) -> Shadow { let mut bridge = walshadow::shadow::BridgeConf::in_dir(&cfg.socket_dir); bridge.socket_path = args.bridge_socket_path(); bridge.library_dir = args.bridge_lib_dir.clone(); + bridge.workers = args.bridge_workers; cfg.bridge = Some(bridge); Shadow::new(cfg) } diff --git a/src/catalog/shadow.rs b/src/catalog/shadow.rs index 2d5825cf..0d8f0c0c 100644 --- a/src/catalog/shadow.rs +++ b/src/catalog/shadow.rs @@ -69,6 +69,10 @@ pub struct BridgeConf { /// Bounds a catalog lock the worker cannot get, which would otherwise hang /// against recovery pub lock_timeout: Duration, + /// `walshadow.bridge_workers`. Each worker serves one request at a time, + /// so this is how many oracle round trips can be in flight. Worker 0 + /// keeps `socket_path`; worker `i` listens on `socket_path.i` + pub workers: usize, } impl BridgeConf { @@ -79,6 +83,7 @@ impl BridgeConf { library_dir: None, io_timeout: Duration::from_secs(30), lock_timeout: Duration::from_secs(1), + workers: 1, } } @@ -99,11 +104,14 @@ impl BridgeConf { "walshadow.socket_path = '{}'\n\ walshadow.database = '{}'\n\ walshadow.io_timeout_ms = {}\n\ - walshadow.lock_timeout_ms = {}\n", + walshadow.lock_timeout_ms = {}\n\ + walshadow.bridge_workers = {}\n", quote_path(&self.socket_path), quote(dbname), self.io_timeout.as_millis(), self.lock_timeout.as_millis(), + self.workers + .clamp(1, crate::ops::bridge::MAX_BRIDGE_WORKERS), )); out } diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 848404a3..a56aa5e0 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -1486,6 +1486,51 @@ fn name_column(mut e: EmitterError, target: &str) -> EmitterError { } /// Build stable leaves before roots borrow them +/// Answer an oracle-routed column locally when the request would be a +/// no-op: every cell is bytes the daemon rendered, and the worker's `String` +/// path wraps each in a `text` that pgch casts straight back. Defaults take +/// the shape `ws_append_default` gives them — NULL under `Nullable`, empty +/// string otherwise. `None` leaves the column in the request +pub(crate) fn literal_column( + buf: &OracleColumnBuf, + target_type: &str, + n_rows: usize, +) -> Option { + if !buf.literal_only() || buf.cells().len() != n_rows { + return None; + } + // `canonical_type` renders these two exactly; anything wrapping a String + // (`LowCardinality`, `Array`) is PG's to build + let nullable = target_type == "Nullable(String)"; + if !nullable && target_type != "String" { + return None; + } + let mut offsets = Vec::with_capacity(n_rows); + let mut data = Vec::new(); + let mut null_map = vec![0u8; if nullable { n_rows } else { 0 }]; + for (i, cell) in buf.cells().iter().enumerate() { + match cell { + OracleCell::Literal(b) => data.extend_from_slice(b), + // `literal_only` admits nothing else + _ => { + if nullable { + null_map[i] = 1; + } + } + } + offsets.push(data.len() as u64); + } + Some(if nullable { + ColumnBuf::NullableString { + offsets, + data, + null_map, + } + } else { + ColumnBuf::String { offsets, data } + }) +} + pub(crate) fn build_leaf( buf: &ColumnBuf, n_rows: usize, @@ -1991,6 +2036,19 @@ crate::atomic_stats! { pub insertbatch_rows_in, pub insertbatch_batches_out, pub inserter_batches_in, + /// Inserter time inside the INSERT round trip (`send_query` through + /// `EndOfStream`), retries included. Against `inserter_pool_size × + /// elapsed` this is CH-side utilization + pub inserter_ch_nanos, + /// Inserter time rebuilding the Native block over the batch's slabs + pub inserter_encode_nanos, + /// Resolver time inside the oracle round trip, retries included. + /// Overlaps the inserters' ClickHouse time, so against + /// `inserter_ch_nanos` it says which stage is the limiter + pub oracle_resolve_nanos, + /// Oracle-routed columns the daemon built itself: rendered cells + /// against a `String` target, which PG would hand straight back + pub oracle_local_columns, } } @@ -2352,6 +2410,30 @@ mod tests { } } + /// A default under a bare `String` is the empty string, matching + /// `ws_append_default`; a target that wraps String is PG's to build + #[test] + fn literal_column_takes_only_a_bare_string_target() { + let mut buf = OracleColumnBuf::new(0, -1); + buf.push(OracleCell::Literal(b"a".to_vec())); + buf.push(OracleCell::Default); + match literal_column(&buf, "String", 2) { + Some(ColumnBuf::String { offsets, data }) => { + assert_eq!(data, b"a"); + assert_eq!(offsets, [1, 1]); + } + other => panic!("got {other:?}"), + } + for reject in ["Array(String)", "LowCardinality(String)", "JSON", "Int32"] { + assert!( + literal_column(&buf, reject, 2).is_none(), + "{reject} needs the worker", + ); + } + // Cell count must match the batch, else offsets would not + assert!(literal_column(&buf, "String", 3).is_none()); + } + #[test] fn new_for_ast_picks_shape_from_chc_type_kind() { let alloc = Allocator::stdlib(); diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 786c1485..8cdf891f 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -15,7 +15,8 @@ use crate::config::ResolvedConfig; use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::ack::AckHandle; -use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow}; +use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow, RowChunk}; +use crate::emit::pipeline::decode::DECODE_CHUNK_BYTES; use crate::emit::route::{RouteSnapshot, RowPolicy}; use crate::mapping::{MappingHandle, TableMapping}; use crate::ops::oracle::render_ext_columns; @@ -27,6 +28,11 @@ use crate::toast::{ use ahash::HashMap; use std::collections::HashSet; +/// Rows per `BatcherMsg::Rows` from the bootstrap drain. Fixed rather than +/// config-driven: bootstrap rows are uniform inserts, and the byte trigger +/// covers the fat-row case +const DRAIN_CHUNK_ROWS: usize = 1024; + /// Completion frontier for `FlushAll` and resume advance #[derive(Debug, Clone, Copy, Default)] pub struct BootstrapDrainOutcome { @@ -40,7 +46,7 @@ pub struct BootstrapDrainOutcome { /// descriptor and mapping re-resolve from frozen per-pass snapshots #[allow(clippy::too_many_arguments)] pub async fn drain( - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver>, catalog: CatalogMap, mapping_handle: MappingHandle, msg_tx: mpsc::Sender, @@ -74,79 +80,84 @@ pub async fn drain( let mut chunk_batch: Vec = Vec::new(); let mut chunk_batch_bytes = 0usize; let mut start_lsn = 0u64; - while let Some(tuple) = rx.recv().await { - let rfn = tuple.rfn; - let source_lsn = tuple.source_lsn; - start_lsn = source_lsn; - - let same = matches!(&open, Some((r, _, _)) if *r == rfn); - let seq = if same { - open.as_ref().expect("same implies open").1 - } else { - if let Some((_, prev_seq, prev_rows)) = open.take() { - ack.placed(prev_seq, prev_rows); - } - let s = next_seq; - next_seq += 1; - ack.register(s, source_lsn); - open = Some((rfn, s, 0)); - s - }; + // Rows coalesce into `BatcherMsg::Rows` on the same dual trigger the + // decode pool uses, so a walk slab costs one channel hop, not one per row + let mut out = RowBuf::default(); + while let Some(slab) = rx.recv().await { + for tuple in slab { + let rfn = tuple.rfn; + let source_lsn = tuple.source_lsn; + start_lsn = source_lsn; + + let same = matches!(&open, Some((r, _, _)) if *r == rfn); + let seq = if same { + open.as_ref().expect("same implies open").1 + } else { + if let Some((_, prev_seq, prev_rows)) = open.take() { + // Every row of the closing seq on the channel before its + // expected count is published + out.flush(&msg_tx).await?; + ack.placed(prev_seq, prev_rows); + } + let s = next_seq; + next_seq += 1; + ack.register(s, source_lsn); + open = Some((rfn, s, 0)); + s + }; - let Some(rel) = catalog.get(rfn.db_node, rfn.rel_node) else { - stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); - continue; - }; + let Some(rel) = catalog.get(rfn.db_node, rfn.rel_node) else { + stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); + continue; + }; - if skip_initial.contains(&rel.rel_name) { - continue; - } + if skip_initial.contains(&rel.rel_name) { + continue; + } - if catalog.is_toast(rfn.db_node, rfn.rel_node) { - if let Some(row) = row_from_columns(tuple, rel.oid) { - chunk_batch_bytes += row.chunk_data.len(); - chunk_batch.push(row); - if chunk_batch.len() >= CHUNK_PUT_BATCH || chunk_batch_bytes >= CHUNK_PUT_BYTES { - flush_chunks(&resolver, &mut chunk_batch).await?; - chunk_batch_bytes = 0; + if catalog.is_toast(rfn.db_node, rfn.rel_node) { + if let Some(row) = row_from_columns(tuple, rel.oid) { + chunk_batch_bytes += row.chunk_data.len(); + chunk_batch.push(row); + if chunk_batch.len() >= CHUNK_PUT_BATCH || chunk_batch_bytes >= CHUNK_PUT_BYTES + { + flush_chunks(&resolver, &mut chunk_batch).await?; + chunk_batch_bytes = 0; + } } + continue; } - continue; - } - - let Some(route) = routes.get(&rel.rel_name).cloned() else { - stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); - continue; - }; - if has_mapped_external_toast(&tuple, &route.mapping) { - if resolver.stores_chunks() { - deferred - .push(tuple) - .await - .map_err(|e| format!("bootstrap: deferred spool: {e}"))?; - stats - .bootstrap_deferred_bytes - .store(deferred.resident_bytes() as u64, Ordering::Relaxed); - stats - .bootstrap_deferred_spool_bytes - .store(deferred.spooled_bytes(), Ordering::Relaxed); + let Some(route) = routes.get(&rel.rel_name).cloned() else { + stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); continue; - } + }; + let mut tuple = tuple; - let permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; + let mut permit = None; + if has_mapped_external_toast(&tuple, &route.mapping) { + if resolver.stores_chunks() { + deferred + .push(tuple) + .await + .map_err(|e| format!("bootstrap: deferred spool: {e}"))?; + stats + .bootstrap_deferred_bytes + .store(deferred.resident_bytes() as u64, Ordering::Relaxed); + stats + .bootstrap_deferred_spool_bytes + .store(deferred.spooled_bytes(), Ordering::Relaxed); + continue; + } + permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; + } render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, permit).await?; + out.push(&msg_tx, seq, rel, route, tuple, permit).await?; bump(&mut open, &mut rows_routed); - continue; } - - let mut tuple = tuple; - render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, None).await?; - bump(&mut open, &mut rows_routed); } + out.flush(&msg_tx).await?; if let Some((_, seq, rows)) = open.take() { ack.placed(seq, rows); } @@ -185,10 +196,11 @@ pub async fn drain( }; let permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, permit).await?; + out.push(&msg_tx, seq, rel, route, tuple, permit).await?; placed += 1; rows_routed += 1; } + out.flush(&msg_tx).await?; replay .finish() .await @@ -213,25 +225,54 @@ fn bump(open: &mut Option<(walrus::pg::walparser::RelFileNode, u64, u64)>, rows_ *rows_routed += 1; } -async fn route_row( - msg_tx: &mpsc::Sender, - seq: u64, - rel: Arc, - route: Arc, - tuple: BackfillTuple, - value_permit: Option, -) -> Result<(), String> { - let committed = tuple.into_committed_insert(); - msg_tx - .send(BatcherMsg::Row(RoutedRow { +/// Coalesces routed rows into one `BatcherMsg::Rows` per +/// [`DECODE_CHUNK_BYTES`]-shaped trigger, the same amortization the streaming +/// decode pool gets. Rows of different seqs may share a chunk; the batcher +/// routes each independently +#[derive(Default)] +struct RowBuf { + rows: Vec, + bytes: usize, +} + +impl RowBuf { + async fn push( + &mut self, + msg_tx: &mpsc::Sender, + seq: u64, + rel: Arc, + route: Arc, + tuple: BackfillTuple, + value_permit: Option, + ) -> Result<(), String> { + let committed = tuple.into_committed_insert(); + self.bytes += committed.decoded.approx_bytes(); + self.rows.push(RoutedRow { seq, rel, route, committed, value_permit: value_permit.map(Arc::new), - })) - .await - .map_err(|_| "bootstrap: batcher channel closed".to_string()) + }); + if self.rows.len() >= DRAIN_CHUNK_ROWS || self.bytes >= DECODE_CHUNK_BYTES { + self.flush(msg_tx).await?; + } + Ok(()) + } + + async fn flush(&mut self, msg_tx: &mpsc::Sender) -> Result<(), String> { + if self.rows.is_empty() { + return Ok(()); + } + self.bytes = 0; + msg_tx + .send(BatcherMsg::Rows(RowChunk { + rows: std::mem::take(&mut self.rows), + permit: None, + })) + .await + .map_err(|_| "bootstrap: batcher channel closed".to_string()) + } } /// Check only columns routed to ClickHouse @@ -348,6 +389,21 @@ fn row_from_columns(mut tuple: BackfillTuple, toast_relid: u32) -> Option) -> Vec { + let mut rows = Vec::new(); + while let Some(msg) = rx.recv().await { + match msg { + BatcherMsg::Rows(chunk) => rows.extend(chunk.rows), + BatcherMsg::Row(r) => rows.push(r), + BatcherMsg::FlushAll(reply) => { + let _ = reply.send(()); + } + } + } + rows + } + use super::*; use crate::backfill::spool::DEFERRED_SPOOL_MEM_MAX; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; @@ -556,13 +612,13 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); for id in 0..3 { - tup_tx.send(tuple(16400, id)).await.unwrap(); + tup_tx.send(vec![tuple(16400, id)]).await.unwrap(); } for id in 0..2 { - tup_tx.send(tuple(16401, id)).await.unwrap(); + tup_tx.send(vec![tuple(16401, id)]).await.unwrap(); } drop(tup_tx); @@ -582,7 +638,7 @@ mod tests { )); let mut by_seq: HashMap = HashMap::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { + for r in collect_rows(&mut msg_rx).await { *by_seq.entry(r.seq).or_default() += 1; } let outcome = drain_task.await.unwrap().unwrap(); @@ -611,12 +667,12 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); // 16400, 16401(unmapped), 16400 → seqs 0,1,2; only 0 and 2 route - tup_tx.send(tuple(16400, 1)).await.unwrap(); - tup_tx.send(tuple(16401, 9)).await.unwrap(); - tup_tx.send(tuple(16400, 2)).await.unwrap(); + tup_tx.send(vec![tuple(16400, 1)]).await.unwrap(); + tup_tx.send(vec![tuple(16401, 9)]).await.unwrap(); + tup_tx.send(vec![tuple(16400, 2)]).await.unwrap(); drop(tup_tx); let stats = Arc::new(EmitterStats::default()); @@ -634,10 +690,11 @@ mod tests { std::collections::HashSet::new(), )); - let mut seqs: Vec = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - seqs.push(r.seq); - } + let seqs: Vec = collect_rows(&mut msg_rx) + .await + .iter() + .map(|r| r.seq) + .collect(); let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 3, "three distinct rfn runs"); assert_eq!(outcome.rows_routed, 2, "unmapped rel routed nothing"); @@ -660,10 +717,10 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); drop(tup_tx); @@ -684,10 +741,7 @@ mod tests { std::collections::HashSet::new(), )); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 1); assert_eq!(outcome.rows_routed, 1); @@ -719,16 +773,16 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); let spool_tmp = tempfile::tempdir().unwrap(); // toast chunk first (its own zero-row seq), then the referring main row tup_tx - .send(toast_chunk_tuple(16500, 1, 0, b"hello")) + .send(vec![toast_chunk_tuple(16500, 1, 0, b"hello")]) .await .unwrap(); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); drop(tup_tx); @@ -750,10 +804,7 @@ mod tests { std::collections::HashSet::new(), )); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 3, "toast seq, main seq, deferred seq"); assert_eq!(outcome.rows_routed, 1); @@ -782,14 +833,14 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); tup_tx - .send(toast_chunk_tuple(16500, 1, 0, b"hello")) + .send(vec![toast_chunk_tuple(16500, 1, 0, b"hello")]) .await .unwrap(); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); @@ -817,10 +868,7 @@ mod tests { mapping.mutate(|m| Arc::make_mut(m).clear()).await; drop(tup_tx); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed, 1, "unmapping must not drop the row"); assert_eq!(rows.len(), 1); diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index 22e05169..87dd3089 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -23,8 +23,8 @@ use crate::config::ResolvedConfig; use crate::emit::ch_emitter::{ColumnBuf, EmitterConfig, EmitterStats, build_leaf, build_root}; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::AckHandle; -use crate::emit::pipeline::batcher::{BatchMeta, InsertBatch}; -use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; +use crate::emit::pipeline::batcher::BatchMeta; +use crate::emit::pipeline::resolver::ResolvedBatch; use crate::schema::RelName; use ahash::{HashMap, HashMapExt}; use std::sync::Arc; @@ -46,7 +46,6 @@ struct Inserter { /// compression are re-read at each batch boundary (a compression change /// reconnects, since the codec is fixed at connect). config_rx: Option>>, - oracle: Option>, } impl Inserter { @@ -84,6 +83,7 @@ impl Inserter { let retry = self.config.retry.clone(); let mut attempt = 0u32; let mut backoff = retry.initial_backoff; + let started = std::time::Instant::now(); reconnect_if_idle(&mut self.client, &self.config, self.last_used).await?; loop { let attempt_result = with_timeout(self.config.insert_timeout, async { @@ -97,6 +97,9 @@ impl Inserter { match attempt_result { Ok(()) => { self.last_used = std::time::Instant::now(); + self.stats + .inserter_ch_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); return Ok(()); } Err(e) if is_retryable(&e) && attempt < retry.max_attempts => { @@ -110,8 +113,13 @@ impl Inserter { } } - async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { - while let Ok(batch) = rx.recv().await { + async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { + while let Ok(ResolvedBatch { + batch, + local, + resolved, + }) = rx.recv().await + { // Live emitter knobs (overlay active): pick up the retry budget and // compression. A compression change needs a fresh client — the codec // is fixed at connect — reconnected here at a batch boundary, never @@ -174,22 +182,17 @@ impl Inserter { .asts .remove(&batch.meta.table_key) .expect("ensure_asts inserted"); + // Columns the resolver built locally stand in for their oracle + // buffers; the rest still splice out of the answered block + let bufs: Vec<&ColumnBuf> = batch + .buffers + .iter() + .enumerate() + .map(|(i, buf)| local.get(i).and_then(Option::as_ref).unwrap_or(buf)) + .collect(); let result = 'send: { - // Keep resolved block alive across every INSERT retry - let resolved = match resolve_oracle_with_retry( - self.oracle.clone(), - self.alloc, - &batch, - &self.config.retry, - &self.stats, - ) - .await - { - Ok(v) => v, - Err(e) => break 'send Err(e), - }; - let leaves: Vec>> = match batch - .buffers + let encode_started = std::time::Instant::now(); + let leaves: Vec>> = match bufs .iter() .map(|buf| build_leaf(buf, batch.n_rows)) .collect::>() @@ -197,8 +200,7 @@ impl Inserter { Ok(v) => v, Err(e) => break 'send Err(e), }; - let roots: Vec>> = match batch - .buffers + let roots: Vec>> = match bufs .iter() .zip(&leaves) .map(|(buf, leaf)| match buf { @@ -234,6 +236,10 @@ impl Inserter { .map_err(Into::into) } }); + self.stats.inserter_encode_nanos.fetch_add( + encode_started.elapsed().as_nanos() as u64, + Ordering::Relaxed, + ); match appended { Ok(()) => self.send_with_retry(&batch.meta.insert_sql, &bb).await, Err(e) => Err(e), @@ -261,73 +267,15 @@ impl Inserter { } } -/// Retry transport failures only -async fn resolve_oracle_with_retry( - oracle: Option>, - alloc: Allocator, - batch: &InsertBatch, - retry: &crate::emit::ch_emitter::RetryConfig, - stats: &EmitterStats, -) -> Result, EmitterError> { - let mut attempt = 0u32; - let mut backoff = retry.initial_backoff; - loop { - match resolve_oracle(oracle.clone(), alloc, batch).await { - Ok(v) => return Ok(v), - Err(e) if e.retryable() && attempt < retry.max_attempts => { - stats.retries_attempted.fetch_add(1, Ordering::Relaxed); - attempt += 1; - backoff_step(&mut backoff, retry.max_backoff).await; - } - Err(e) => return Err(EmitterError::Type(e.to_string())), - } - } -} - -async fn resolve_oracle( - oracle: Option>, - alloc: Allocator, - batch: &InsertBatch, -) -> Result, OracleError> { - let columns: Vec> = batch - .buffers - .iter() - .enumerate() - .filter_map(|(i, buf)| match buf { - ColumnBuf::Oracle(o) => Some(OracleRequestColumn { - ordinal: i as u32, - name: &batch.meta.columns[i].name, - target_type: &batch.meta.columns[i].type_repr, - buf: o, - }), - _ => None, - }) - .collect(); - if columns.is_empty() { - return Ok(None); - } - let oracle = oracle.ok_or_else(|| { - OracleError::Absent(format!( - "{} needs the shadow oracle, which this pipeline has none of", - batch.meta.table_key - )) - })?; - oracle - .encode_batch(&columns, batch.n_rows, alloc) - .await - .map(Some) -} - pub(crate) struct PoolOptions { pub config_rx: Option>>, - pub oracle: Option>, } /// Connect `n` inserters and spawn drain loops pub(crate) async fn spawn_pool( n: usize, config: &EmitterConfig, - rx: async_channel::Receiver, + rx: async_channel::Receiver, ack: AckHandle, stats: Arc, fatal: Fatal, @@ -345,7 +293,6 @@ pub(crate) async fn spawn_pool( ack: ack.clone(), stats: stats.clone(), config_rx: options.config_rx.clone(), - oracle: options.oracle.clone(), }; let rx = rx.clone(); let fatal = fatal.clone(); diff --git a/src/emit/pipeline/mod.rs b/src/emit/pipeline/mod.rs index 3a28e75c..fcc58788 100644 --- a/src/emit/pipeline/mod.rs +++ b/src/emit/pipeline/mod.rs @@ -18,6 +18,7 @@ pub mod inserter; pub mod plan_spool; pub mod planner; pub mod reorder; +pub mod resolver; pub mod tail; use std::sync::Arc; diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs new file mode 100644 index 00000000..63785687 --- /dev/null +++ b/src/emit/pipeline/resolver.rs @@ -0,0 +1,384 @@ +//! Oracle resolve, off the inserters' critical path. +//! +//! An inserter that resolves its own batch leaves its ClickHouse connection +//! idle for the whole round trip, and the oracle idle for the whole INSERT: +//! throughput is `1 / (T_oracle + T_ch / P)` where it could be +//! `1 / max(T_oracle / W, T_ch / P)`. This stage sits between batcher and +//! inserter pool so the two overlap, `W` wide to match the bridge — that is +//! how many requests the shadow answers at once. +//! +//! Costs one extra resident batch per inserter, which is what a queue deep +//! enough to keep every inserter fed takes. +//! +//! A batch with no oracle column crosses unchanged, without touching the +//! bridge. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use clickhouse_c::Allocator; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use crate::ch::{EmitterError, backoff_step}; +use crate::config::ResolvedConfig; +use crate::emit::ch_emitter::{ColumnBuf, EmitterStats, RetryConfig, literal_column}; +use crate::emit::pipeline::Fatal; +use crate::emit::pipeline::batcher::InsertBatch; +use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; + +/// A sealed batch past the oracle. `local` is by ordinal, `Some` where the +/// daemon built an oracle-routed column itself and left it out of the +/// request; the inserter splices from there instead of from `resolved` +pub(crate) struct ResolvedBatch { + pub batch: InsertBatch, + pub local: Vec>, + pub resolved: Option, +} + +/// `None` oracle resolves nothing, which a pipeline with no oracle column +/// and a pipeline that has one but no shadow both look like; the second +/// fails at its first oracle batch +#[derive(Clone)] +pub(crate) struct ResolverOptions { + pub oracle: Option>, + /// Boot budget for oracle transport retries, re-read from `config_rx` + pub retry: RetryConfig, + pub stats: Arc, + pub fatal: Fatal, + pub config_rx: Option>>, +} + +/// Spawn `n` resolvers over the shared batch queue. Drops the sender it was +/// handed, so the inserters' queue closes once every resolver exits +pub(crate) fn spawn_pool( + n: usize, + rx: async_channel::Receiver, + tx: async_channel::Sender, + opts: ResolverOptions, +) -> Vec> { + (0..n.max(1)) + .map(|_| tokio::spawn(run(rx.clone(), tx.clone(), opts.clone()))) + .collect() +} + +async fn run( + rx: async_channel::Receiver, + tx: async_channel::Sender, + opts: ResolverOptions, +) { + let ResolverOptions { + oracle, + mut retry, + stats, + fatal, + config_rx, + } = opts; + let alloc = Allocator::global(&mimalloc::MiMalloc); + while let Ok(batch) = rx.recv().await { + if let Some(rx) = config_rx.as_ref() { + retry.max_attempts = rx.borrow().retry_max_attempts; + } + let local = local_columns(&batch, &stats); + let started = std::time::Instant::now(); + let resolved = + match resolve_oracle_with_retry(&oracle, alloc, &batch, &local, &retry, &stats).await { + Ok(v) => v, + Err(e) => { + // Seq stays unacknowledged, so a restart replays it + fatal.set(format!("oracle resolve: {e}")); + return; + } + }; + stats + .oracle_resolve_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + if tx + .send(ResolvedBatch { + batch, + local, + resolved, + }) + .await + .is_err() + { + return; + } + } +} + +/// Oracle columns the daemon answers itself, by ordinal. Empty when the +/// batch has no oracle column at all, which is the common shape +fn local_columns(batch: &InsertBatch, stats: &EmitterStats) -> Vec> { + if !batch + .buffers + .iter() + .any(|b| matches!(b, ColumnBuf::Oracle(_))) + { + return Vec::new(); + } + let taken: Vec> = batch + .buffers + .iter() + .enumerate() + .map(|(i, buf)| match buf { + ColumnBuf::Oracle(o) => { + literal_column(o, &batch.meta.columns[i].type_repr, batch.n_rows) + } + _ => None, + }) + .collect(); + let n = taken.iter().filter(|c| c.is_some()).count() as u64; + if n > 0 { + stats.oracle_local_columns.fetch_add(n, Ordering::Relaxed); + } + taken +} + +/// Retry transport failures only +async fn resolve_oracle_with_retry( + oracle: &Option>, + alloc: Allocator, + batch: &InsertBatch, + local: &[Option], + retry: &RetryConfig, + stats: &EmitterStats, +) -> Result, EmitterError> { + let mut attempt = 0u32; + let mut backoff = retry.initial_backoff; + loop { + match resolve_oracle(oracle, alloc, batch, local).await { + Ok(v) => return Ok(v), + Err(e) if e.retryable() && attempt < retry.max_attempts => { + stats.retries_attempted.fetch_add(1, Ordering::Relaxed); + attempt += 1; + backoff_step(&mut backoff, retry.max_backoff).await; + } + Err(e) => return Err(EmitterError::Type(e.to_string())), + } + } +} + +async fn resolve_oracle( + oracle: &Option>, + alloc: Allocator, + batch: &InsertBatch, + local: &[Option], +) -> Result, OracleError> { + let columns: Vec> = batch + .buffers + .iter() + .enumerate() + .filter_map(|(i, buf)| match buf { + ColumnBuf::Oracle(o) if local.get(i).is_none_or(Option::is_none) => { + Some(OracleRequestColumn { + ordinal: i as u32, + name: &batch.meta.columns[i].name, + target_type: &batch.meta.columns[i].type_repr, + buf: o, + }) + } + _ => None, + }) + .collect(); + if columns.is_empty() { + return Ok(None); + } + let oracle = oracle.as_ref().ok_or_else(|| { + OracleError::Absent(format!( + "{} needs the shadow oracle, which this pipeline has none of", + batch.meta.table_key + )) + })?; + oracle + .encode_batch(&columns, batch.n_rows, alloc) + .await + .map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decode::heap_decoder::{ + ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, + }; + use crate::emit::pipeline::batcher::{BatcherConfig, BatcherMsg, RoutedRow}; + use crate::emit::route::RouteSnapshot; + use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; + use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; + use std::time::Duration; + use tokio::sync::{mpsc, oneshot}; + use walrus::pg::walparser::RelFileNode; + + /// An oracle-routed column the local matrix does not cover, rendered by + /// the daemon: the shape a PostGIS 2-D point takes + const GEOM_OID: u32 = 999_999; + + fn wkt_row(seq: u64, text: Option<&str>) -> RoutedRow { + let rel = Arc::new(RelDescriptor { + rfn: RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node: 16385, + }, + oid: 16385, + toast_oid: 0, + namespace_oid: 2200, + rel_name: RelName::new("public", "g"), + kind: 'r', + persistence: 'p', + replident: ReplIdent::Default { pk_attnums: None }, + attributes: vec![RelAttr { + attnum: 1, + name: "geom".into(), + type_oid: GEOM_OID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "geometry".into(), + type_byval: false, + type_len: -1, + type_align: 'i', + type_storage: 'x', + missing_text: None, + }], + }); + let route = RouteSnapshot::freeze( + Arc::new(TableMapping { + target: TableTarget::new("default", "g"), + columns: vec![ColumnMapping { + src_attnum: 1, + target_name: "geom".into(), + target_type: "Nullable(String)".into(), + }], + }), + Arc::default(), + Default::default(), + ); + RoutedRow { + seq, + rel, + route, + committed: CommittedTuple { + decoded: DecodedHeap { + rfn: RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node: 16385, + }, + xid: 7, + source_lsn: 0x1000 + seq, + op: HeapOp::Insert, + new: Some(DecodedTuple { + columns: vec![text.map(|t| ColumnValue::Text(t.into()))], + partial: false, + }), + old: None, + }, + commit_ts: 0, + commit_lsn: (seq + 1) * 100, + }, + value_permit: None, + } + } + + /// One batch of the rows, sealed through the real batcher: `InsertBatch` + /// is only reachable that way, and its cell tags are what the encoder + /// chose rather than what a test asserted + async fn seal(rows: Vec) -> InsertBatch { + let (msg_tx, msg_rx) = mpsc::channel(64); + let (batches_tx, batches_rx) = async_channel::bounded(64); + let fatal = Fatal::new(); + let handle = crate::emit::pipeline::batcher::spawn( + msg_rx, + batches_tx, + BatcherConfig { + row_budget: 1_000, + byte_budget: 1 << 30, + flush_timeout: Duration::from_secs(3600), + }, + Allocator::stdlib(), + fatal.clone(), + Arc::new(EmitterStats::default()), + None, + ); + for r in rows { + msg_tx.send(BatcherMsg::Row(r)).await.expect("send row"); + } + let (reply_tx, reply_rx) = oneshot::channel(); + msg_tx + .send(BatcherMsg::FlushAll(reply_tx)) + .await + .expect("send flush"); + reply_rx.await.expect("flush ack"); + let batch = batches_rx.recv().await.expect("one batch"); + drop(msg_tx); + handle.await.expect("batcher task"); + assert!(fatal.message().is_none()); + batch + } + + /// Cells the daemon rendered against a `String` target never reach the + /// bridge: the same batch resolves with no oracle at all, and fails + /// without the local path + #[tokio::test] + async fn rendered_cells_against_a_string_target_stay_local() { + let batch = seal(vec![ + wkt_row(0, Some("POINT(1 2)")), + wkt_row(0, None), + wkt_row(0, Some("POINT(3 4)")), + ]) + .await; + assert!( + matches!(batch.buffers[0], ColumnBuf::Oracle(_)), + "premise: an uncovered source type routes to the oracle", + ); + + let stats = EmitterStats::default(); + let local = local_columns(&batch, &stats); + assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 1); + let Some(ColumnBuf::NullableString { + offsets, + data, + null_map, + }) = local[0].as_ref() + else { + panic!("column not built locally: {:?}", local[0]); + }; + assert_eq!(data, b"POINT(1 2)POINT(3 4)"); + assert_eq!(offsets, &[10, 10, 20], "absent cell holds its offset"); + assert_eq!(null_map, &[0, 1, 0]); + + let alloc = Allocator::stdlib(); + assert!( + resolve_oracle(&None, alloc, &batch, &local) + .await + .expect("no request to make") + .is_none(), + ); + // Same batch without the local build has to ask someone + assert!(matches!( + resolve_oracle(&None, alloc, &batch, &[]).await, + Err(OracleError::Absent(_)), + )); + } + + /// A cell PG must convert keeps the whole column in the request, even + /// when its neighbours are rendered + #[tokio::test] + async fn one_unrendered_cell_keeps_the_column_remote() { + let mut rows = vec![wkt_row(0, Some("POINT(1 2)"))]; + let mut raw = wkt_row(0, None); + raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + type_oid: GEOM_OID, + raw: vec![1, 2, 3], + }); + rows.push(raw); + let batch = seal(rows).await; + + let stats = EmitterStats::default(); + let local = local_columns(&batch, &stats); + assert!(local[0].is_none()); + assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 0); + } +} diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 9348da00..1b1d4a85 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -23,6 +23,7 @@ use crate::emit::ch_emitter::{EmitterConfig, EmitterStats}; use crate::emit::pipeline::ack::{self, AckHandle}; use crate::emit::pipeline::batcher::{self, BatcherConfig, BatcherMsg, InsertBatch}; use crate::emit::pipeline::inserter; +use crate::emit::pipeline::resolver::{self, ResolvedBatch}; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; use crate::pos::{EmitterAck, Monotone}; use ahash::{HashMap, HashMapExt}; @@ -31,6 +32,7 @@ use ahash::{HashMap, HashMapExt}; pub struct TailParts { collector: JoinHandle<()>, batcher: JoinHandle<()>, + resolvers: Vec>, inserters: Vec>, } @@ -40,6 +42,9 @@ impl TailParts { /// channel close and this hangs. pub async fn join(self) { let _ = self.batcher.await; + for h in self.resolvers { + let _ = h.await; + } for h in self.inserters { let _ = h.await; } @@ -146,6 +151,7 @@ pub fn spawn_null( TailParts { collector, batcher, + resolvers: Vec::new(), inserters: Vec::new(), }, ) @@ -171,21 +177,37 @@ pub async fn spawn_with_config( // enqueued before it let (msg_tx, msg_rx) = mpsc::channel::(256); let (batches_tx, batches_rx) = async_channel::bounded::((n * 2).max(4)); + // One resolved batch per inserter is what keeps every one of them fed + let (resolved_tx, resolved_rx) = async_channel::bounded::(n); let inserters = inserter::spawn_pool( n, emitter, - batches_rx, + resolved_rx, ack.clone(), stats.clone(), fatal.clone(), inserter::PoolOptions { config_rx: config_rx.clone(), - oracle, }, ) .await?; + // As wide as the bridge: a narrower pool leaves shadow workers idle, a + // wider one only queues on their sockets + let resolvers = resolver::spawn_pool( + oracle.as_ref().map_or(1, |o| o.concurrency()), + batches_rx, + resolved_tx, + resolver::ResolverOptions { + oracle, + retry: emitter.retry.clone(), + stats: stats.clone(), + fatal: fatal.clone(), + config_rx: config_rx.clone(), + }, + ); + // Boot fallback for the batcher; the live path re-reads from `config_rx`. let flush_timeout = if emitter.flush_timeout.is_zero() { DEFAULT_PIPELINE_FLUSH @@ -212,6 +234,7 @@ pub async fn spawn_with_config( TailParts { collector, batcher, + resolvers, inserters, }, )) diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index bbb517b6..57cb1c9a 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -10,7 +10,7 @@ use std::io; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -27,11 +27,17 @@ pub const PROJECTION_VERSION: u32 = 1; /// Match `WS_MAX_REQUEST_BYTES` pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024; +/// `len: u32be` then `op: u8`. Request builders reserve this much leading +/// room so the bridge patches the prefix in place and one `write_all` ships +/// the frame, rather than reallocating and copying the whole payload +pub const FRAME_PREFIX_BYTES: usize = 5; /// Whole-catalog `pg_type` text output is the largest response in practice const MAX_RESPONSE_BYTES: usize = 256 * 1024 * 1024; /// Matches `WS_MAX_SCAN_OIDS`. A longer list is the caller's to chunk, since /// only the caller knows whether the chunks share a replay position pub const MAX_SCAN_OIDS: usize = 65536; +/// Matches `WS_MAX_WORKERS`, the ceiling on `walshadow.bridge_workers` +pub const MAX_BRIDGE_WORKERS: usize = 8; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] @@ -50,6 +56,13 @@ impl Op { fn slot(self) -> usize { self as usize - 1 } + + /// Catalog reads pin to worker 0. `SCAN` answers off a replay position + /// it reports back, and `HELLO` establishes the identity every other + /// worker is then checked against, so neither may drift between sockets + fn pinned(self) -> bool { + matches!(self, Op::Hello | Op::Scan) + } } /// Catalogs the overlay scan covers. Ids are wire values; never renumber @@ -170,10 +183,20 @@ crate::atomic_stats! { /// reads answer these off SQL instead; overlay reads fail pub scan_replay_moved, pub native_bytes, + /// Bridge sockets the pool holds, one per shadow-side worker + pub pool_size, /// Per-op, indexed by [`OP_LABELS`] pub requests: [AtomicU64; OP_COUNT], pub errors: [AtomicU64; OP_COUNT], pub request_nanos: [AtomicU64; OP_COUNT], + /// Queued behind another caller's request on the one socket. Against + /// `service_nanos` this is what says whether the worker is the + /// limiter or the funnel in front of it is + pub lock_wait_nanos: [AtomicU64; OP_COUNT], + /// Wire time with the socket held: worker conversion plus transfer + pub service_nanos: [AtomicU64; OP_COUNT], + pub request_bytes: [AtomicU64; OP_COUNT], + pub response_bytes: [AtomicU64; OP_COUNT], } } @@ -203,11 +226,23 @@ impl BridgeStats { } } +/// One shadow-side worker's socket. A worker serves one request at a time, +/// so the mutex is the worker, not an artefact of sharing #[derive(Debug)] -pub struct Bridge { +struct Slot { path: PathBuf, conn: Mutex>, - /// Set by the first successful `HELLO`; later dials must match it +} + +#[derive(Debug)] +pub struct Bridge { + /// Slot 0 is `walshadow.socket_path`, slot `i` is `socket_path.i`. + /// Stateless ops round-robin; [`Op::pinned`] ops stay on slot 0 + slots: Vec, + next: AtomicUsize, + /// Set by the first successful `HELLO`; later dials, on any slot, must + /// match it. A worker that came back a different build means a mixed + /// install and must fail closed info: OnceLock, pub stats: Arc, } @@ -217,19 +252,54 @@ impl Bridge { /// mismatch is refused rather than negotiated: the daemon would misparse /// the projections pub async fn connect(path: impl AsRef) -> Result { + Self::connect_pooled(path, 1).await + } + + /// One socket per shadow-side worker, matching + /// `walshadow.bridge_workers`. Every slot must answer: a pool short of + /// the configured width is a half-started shadow, and silently running + /// narrower would hide it + pub async fn connect_pooled( + path: impl AsRef, + workers: usize, + ) -> Result { + let base = path.as_ref(); + let slots: Vec = (0..workers.clamp(1, MAX_BRIDGE_WORKERS)) + .map(|i| Slot { + path: if i == 0 { + base.to_owned() + } else { + PathBuf::from(format!("{}.{i}", base.display())) + }, + conn: Mutex::new(None), + }) + .collect(); let bridge = Self { - path: path.as_ref().to_owned(), - conn: Mutex::new(None), + slots, + next: AtomicUsize::new(0), info: OnceLock::new(), stats: Arc::new(BridgeStats::default()), }; - let stream = bridge.dial().await?; - *bridge.conn.lock().await = Some(stream); + // Slot 0 first, so its HELLO is the identity the rest are checked + // against rather than whichever worker happened to answer first + for slot in &bridge.slots { + let stream = bridge.dial(slot).await?; + *slot.conn.lock().await = Some(stream); + } + bridge + .stats + .pool_size + .store(bridge.slots.len() as u64, Ordering::Relaxed); Ok(bridge) } + /// `walshadow.socket_path`, ie slot 0 pub fn path(&self) -> &Path { - &self.path + &self.slots[0].path + } + + pub fn pool_size(&self) -> usize { + self.slots.len() } /// `None` before the first successful `HELLO`, which [`connect`](Self::connect) @@ -244,13 +314,14 @@ impl Bridge { /// `pg_last_wal_replay_lsn` by shared-memory read, one round trip pub async fn replay_lsn(&self) -> Result { - let body = self.call(Op::ReplayLsn, &[]).await?; + let body = self.call(Op::ReplayLsn, request_frame(0)).await?; Cursor::at(&body, 1).u64() } - /// Return response remainder as one locally framed Native block - pub async fn encode_native(&self, payload: &[u8]) -> Result { - let frame = self.call(Op::EncodeNative, payload).await?; + /// Return response remainder as one locally framed Native block. + /// `frame` carries [`FRAME_PREFIX_BYTES`] of unwritten leading room + pub async fn encode_native(&self, frame: Vec) -> Result { + let frame = self.call(Op::EncodeNative, frame).await?; self.stats .native_bytes .fetch_add((frame.len() - 1) as u64, Ordering::Relaxed); @@ -270,15 +341,15 @@ impl Bridge { top_xid: u32, oids: &[u32], ) -> Result { - let mut payload = Vec::with_capacity(9 + oids.len() * 4); - payload.push(cat as u8); - payload.extend_from_slice(&top_xid.to_be_bytes()); - payload.extend_from_slice(&(oids.len() as u32).to_be_bytes()); + let mut frame = request_frame(9 + oids.len() * 4); + frame.push(cat as u8); + frame.extend_from_slice(&top_xid.to_be_bytes()); + frame.extend_from_slice(&(oids.len() as u32).to_be_bytes()); for oid in oids { - payload.extend_from_slice(&oid.to_be_bytes()); + frame.extend_from_slice(&oid.to_be_bytes()); } - let body = self.call(Op::Scan, &payload).await?; + let body = self.call(Op::Scan, frame).await?; let mut c = Cursor::at(&body, 1); let replay_lsn_start = c.u64()?; let replay_lsn_end = c.u64()?; @@ -367,10 +438,13 @@ impl Bridge { /// Fresh socket plus `HELLO`. Takes no connection lock, so /// [`call`](Self::call) may hold one across it - async fn dial(&self) -> Result { - let mut stream = UnixStream::connect(&self.path).await?; + async fn dial(&self, slot: &Slot) -> Result { + let mut stream = UnixStream::connect(&slot.path).await?; + widen_sockbufs(&stream); let started = Instant::now(); - let res = round_trip(&mut stream, Op::Hello, &[]).await; + let mut hello = request_frame(0); + patch_frame(&mut hello, Op::Hello); + let res = round_trip(&mut stream, &hello).await; self.record(Op::Hello, started, &res); let body = res?; @@ -400,11 +474,13 @@ impl Bridge { Ok(stream) } - async fn call(&self, op: Op, payload: &[u8]) -> Result, BridgeError> { + /// `frame` carries [`FRAME_PREFIX_BYTES`] of unwritten leading room, + /// patched here rather than copied into a second buffer + async fn call(&self, op: Op, mut frame: Vec) -> Result, BridgeError> { let started = Instant::now(); // Refuse before the socket sees it: the worker answers a frame this // size by closing, and a healthy connection must not pay for that - let len = payload.len() + 1; + let len = frame.len() - FRAME_PREFIX_BYTES + 1; if len > MAX_REQUEST_BYTES { let res = Err(BridgeError::RequestTooLarge { len, @@ -413,9 +489,18 @@ impl Bridge { self.record(op, started, &res); return res; } - let mut guard = self.conn.lock().await; + patch_frame(&mut frame, op); + let stat = op.slot(); + self.stats.request_bytes[stat].fetch_add(frame.len() as u64, Ordering::Relaxed); + let slot = self.pick(op); + let mut guard = slot.conn.lock().await; + let held = Instant::now(); + self.stats.lock_wait_nanos[stat].fetch_add( + held.duration_since(started).as_nanos() as u64, + Ordering::Relaxed, + ); let mut res = match guard.as_mut() { - Some(stream) => round_trip(stream, op, payload).await, + Some(stream) => round_trip(stream, &frame).await, None => Err(BridgeError::Io(io::Error::new( io::ErrorKind::NotConnected, "bridge disconnected", @@ -426,9 +511,9 @@ impl Bridge { if is_transport_error(&res) { *guard = None; self.stats.reconnects.fetch_add(1, Ordering::Relaxed); - match self.dial().await { + match self.dial(slot).await { Ok(mut stream) => { - res = round_trip(&mut stream, op, payload).await; + res = round_trip(&mut stream, &frame).await; if !is_transport_error(&res) { *guard = Some(stream); } @@ -437,10 +522,26 @@ impl Bridge { } } drop(guard); + self.stats.service_nanos[stat] + .fetch_add(held.elapsed().as_nanos() as u64, Ordering::Relaxed); + if let Ok(body) = &res { + self.stats.response_bytes[stat].fetch_add(body.len() as u64, Ordering::Relaxed); + } self.record(op, started, &res); res } + /// Round-robin over the pool, except for ops pinned to worker 0. + /// `ENCODE_NATIVE` is stateless and read-only, so any worker answers any + /// request + fn pick(&self, op: Op) -> &Slot { + if op.pinned() || self.slots.len() == 1 { + return &self.slots[0]; + } + let i = self.next.fetch_add(1, Ordering::Relaxed) % self.slots.len(); + &self.slots[i] + } + fn record(&self, op: Op, started: Instant, res: &Result, BridgeError>) { let slot = op.slot(); self.stats.requests[slot].fetch_add(1, Ordering::Relaxed); @@ -463,19 +564,50 @@ fn is_transport_error(res: &Result, BridgeError>) -> bool { matches!(res, Err(e) if e.is_transport()) } -async fn round_trip( - stream: &mut UnixStream, - op: Op, - payload: &[u8], -) -> Result, BridgeError> { - let len = payload.len() + 1; - // One write: a peer that dribbles a partial frame is what the worker's - // io_timeout_ms exists to bound, and the daemon must not be that peer - let mut frame = Vec::with_capacity(4 + len); - frame.extend_from_slice(&(len as u32).to_be_bytes()); - frame.push(op as u8); - frame.extend_from_slice(payload); - stream.write_all(&frame).await?; +/// Matches `WS_SOCKBUF_BYTES` in `pgext/worker.c`. A multi-megabyte frame +/// against default socket buffers costs hundreds of `EAGAIN` round trips +/// on each side, so both ends ask for a wide window +const SOCKBUF_BYTES: libc::c_int = 4 * 1024 * 1024; + +/// Advisory: a kernel that refuses leaves the default, which only costs +/// more wakeups. The worker widens its own end on accept +fn widen_sockbufs(stream: &UnixStream) { + use std::os::fd::AsRawFd; + let fd = stream.as_raw_fd(); + let want = SOCKBUF_BYTES; + for opt in [libc::SO_RCVBUF, libc::SO_SNDBUF] { + // SAFETY: `fd` is owned by `stream` and outlives the call; `want` is + // a live `c_int` of the length passed + unsafe { + libc::setsockopt( + fd, + libc::SOL_SOCKET, + opt, + (&raw const want).cast(), + std::mem::size_of_val(&want) as libc::socklen_t, + ); + } + } +} + +/// Leading room for the length + opcode prefix, unwritten until the bridge +/// knows both +pub fn request_frame(payload_capacity: usize) -> Vec { + let mut v = Vec::with_capacity(FRAME_PREFIX_BYTES + payload_capacity); + v.resize(FRAME_PREFIX_BYTES, 0); + v +} + +fn patch_frame(frame: &mut [u8], op: Op) { + let len = (frame.len() - FRAME_PREFIX_BYTES + 1) as u32; + frame[..4].copy_from_slice(&len.to_be_bytes()); + frame[4] = op as u8; +} + +/// One write: a peer that dribbles a partial frame is what the worker's +/// io_timeout_ms exists to bound, and the daemon must not be that peer +async fn round_trip(stream: &mut UnixStream, frame: &[u8]) -> Result, BridgeError> { + stream.write_all(frame).await?; stream.flush().await?; let mut hdr = [0u8; 4]; @@ -499,9 +631,13 @@ async fn round_trip( /// Connect with a wall-clock budget while shadow reaches consistency. /// Matches catalog's /// [`with_transient_retry`](crate::catalog::shadow_catalog::with_transient_retry) shape -pub async fn connect_with_budget(path: &Path, budget: Duration) -> Result { +pub async fn connect_with_budget( + path: &Path, + workers: usize, + budget: Duration, +) -> Result { let deadline = tokio::time::Instant::now() + budget; - (|| Bridge::connect(path)) + (|| Bridge::connect_pooled(path, workers)) .retry( ExponentialBuilder::default() .with_min_delay(Duration::from_millis(100)) @@ -948,7 +1084,9 @@ mod tests { ]); let bridge = Bridge::connect(&path).await.unwrap(); - let out = bridge.encode_native(&[0u8; 8]).await.unwrap(); + let mut req = request_frame(8); + req.extend_from_slice(&[0u8; 8]); + let out = bridge.encode_native(req).await.unwrap(); assert_eq!(out.bytes(), &native[..]); assert_eq!( bridge.stats.native_bytes.load(Ordering::Relaxed), @@ -1042,8 +1180,9 @@ mod tests { let (_tmp, path) = spawn_worker(vec![Some(hello_body(PROTO_VERSION, PROJECTION_VERSION))]); let bridge = Bridge::connect(&path).await.unwrap(); - let huge = vec![0u8; MAX_REQUEST_BYTES]; - let err = bridge.encode_native(&huge).await.unwrap_err(); + let mut huge = request_frame(MAX_REQUEST_BYTES); + huge.resize(FRAME_PREFIX_BYTES + MAX_REQUEST_BYTES, 0); + let err = bridge.encode_native(huge).await.unwrap_err(); assert!( matches!(err, BridgeError::RequestTooLarge { .. }), "got {err:?}" diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 1a8ad34b..758b04e7 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -78,6 +78,16 @@ pub struct MetricsSnapshot { pub bootstrap_deferred_bytes: u64, /// Encoded bytes in the bootstrap TOAST-deferred spool file pub bootstrap_deferred_spool_bytes: u64, + /// Bootstrap pump stage attribution. Live while a greenfield bootstrap + /// runs, then frozen at its final values for the rest of the session + pub bootstrap_bytes_tapped: u64, + pub bootstrap_pages_walked: u64, + pub bootstrap_tuples_emitted: u64, + pub bootstrap_files_walked: u64, + pub bootstrap_files_skipped_unmapped: u64, + pub bootstrap_decode_seconds: f64, + pub bootstrap_tap_seconds: f64, + pub bootstrap_channel_block_seconds: f64, pub spill_evictions_total: u64, pub xacts_committed_total: u64, pub xacts_aborted_total: u64, @@ -150,8 +160,12 @@ pub struct MetricsSnapshot { pub insertbatch_rows_in_total: u64, pub insertbatch_batches_out_total: u64, pub inserter_batches_in_total: u64, + pub inserter_ch_seconds_total: f64, + pub inserter_encode_seconds_total: f64, + pub oracle_resolve_seconds_total: f64, pub process_cpu_seconds_total: f64, pub process_resident_memory_bytes: u64, + pub oracle_local_columns_total: u64, pub oracle_blocks_total: u64, pub oracle_rows_total: u64, pub oracle_cells_total: u64, @@ -165,6 +179,12 @@ pub struct MetricsSnapshot { pub bridge_requests_by_op: [u64; 4], pub bridge_errors_by_op: [u64; 4], pub bridge_request_nanos_by_op: [u64; 4], + /// Queued behind another caller on the single bridge socket + pub bridge_lock_wait_nanos_by_op: [u64; 4], + /// Wire time with the socket held + pub bridge_service_nanos_by_op: [u64; 4], + pub bridge_request_bytes_by_op: [u64; 4], + pub bridge_response_bytes_by_op: [u64; 4], pub bridge_reconnects_total: u64, pub bridge_scan_rows_total: u64, pub bridge_scan_subtrans_mismatch_total: u64, @@ -573,6 +593,36 @@ pub fn render(snap: &MetricsSnapshot) -> String { "gauge", snap.bootstrap_deferred_spool_bytes, ), + ( + "walshadow_bootstrap_bytes_tapped_total", + "Backup body bytes the bootstrap pump handed to the page-walk sink.", + "counter", + snap.bootstrap_bytes_tapped, + ), + ( + "walshadow_bootstrap_pages_walked_total", + "8 KiB heap pages the bootstrap walk framed.", + "counter", + snap.bootstrap_pages_walked, + ), + ( + "walshadow_bootstrap_tuples_emitted_total", + "Live tuples the bootstrap walk decoded off backup pages.", + "counter", + snap.bootstrap_tuples_emitted, + ), + ( + "walshadow_bootstrap_files_walked_total", + "User-heap segments the bootstrap walk decoded.", + "counter", + snap.bootstrap_files_walked, + ), + ( + "walshadow_bootstrap_files_skipped_unmapped_total", + "User-heap segments declined at begin because no mapped relation owns them; their bytes drain unread.", + "counter", + snap.bootstrap_files_skipped_unmapped, + ), ( "walshadow_spill_evictions_total", "Total evictions in→spill since daemon start.", @@ -784,6 +834,12 @@ pub fn render(snap: &MetricsSnapshot) -> String { "gauge", snap.process_resident_memory_bytes, ), + ( + "walshadow_oracle_local_columns_total", + "Oracle-routed columns the daemon built itself: already-rendered cells against a String target, which PG would hand straight back.", + "counter", + snap.oracle_local_columns_total, + ), ( "walshadow_oracle_blocks_total", "Partial Native blocks the walshadow extension returned and the daemon validated.", @@ -1278,12 +1334,47 @@ pub fn render(snap: &MetricsSnapshot) -> String { for (op, v) in OP_LABELS.iter().zip(snap.bridge_errors_by_op) { writeln!(s, "{name}{{op=\"{op}\"}} {v}").unwrap(); } - let name = "walshadow_bridge_request_seconds_total"; - writeln!(s, "# HELP {name} Wall time spent in bridge round trips.").unwrap(); - writeln!(s, "# TYPE {name} counter").unwrap(); - for (op, v) in OP_LABELS.iter().zip(snap.bridge_request_nanos_by_op) { - let secs = v as f64 / 1e9; - writeln!(s, "{name}{{op=\"{op}\"}} {secs}").unwrap(); + for (name, help, nanos) in [ + ( + "walshadow_bridge_request_seconds_total", + "Wall time spent in bridge round trips.", + snap.bridge_request_nanos_by_op, + ), + ( + "walshadow_bridge_lock_wait_seconds_total", + "Wall time bridge callers spent queued for the socket. Against bridge_service_seconds this says whether the worker or the funnel in front of it is the limiter.", + snap.bridge_lock_wait_nanos_by_op, + ), + ( + "walshadow_bridge_service_seconds_total", + "Wall time on the wire with the bridge socket held: worker conversion plus transfer.", + snap.bridge_service_nanos_by_op, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + for (op, v) in OP_LABELS.iter().zip(nanos) { + let secs = v as f64 / 1e9; + writeln!(s, "{name}{{op=\"{op}\"}} {secs}").unwrap(); + } + } + for (name, help, bytes) in [ + ( + "walshadow_bridge_request_bytes_total", + "Request frame bytes written to the bridge socket.", + snap.bridge_request_bytes_by_op, + ), + ( + "walshadow_bridge_response_bytes_total", + "Response frame bytes read back off the bridge socket.", + snap.bridge_response_bytes_by_op, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + for (op, v) in OP_LABELS.iter().zip(bytes) { + writeln!(s, "{name}{{op=\"{op}\"}} {v}").unwrap(); + } } } @@ -1339,6 +1430,43 @@ pub fn render(snap: &MetricsSnapshot) -> String { writeln!(s, "# TYPE {name} counter").unwrap(); writeln!(s, "{name} {:.3}", snap.desc_capture_seconds_total).unwrap(); + for (name, help, secs) in [ + ( + "walshadow_bootstrap_decode_seconds_total", + "Cumulative CPU inside the bootstrap page walk, tuple decode included.", + snap.bootstrap_decode_seconds, + ), + ( + "walshadow_bootstrap_tap_seconds_total", + "Cumulative time bootstrap tap readers spent inside the sink: page framing, decode, channel send. Against bootstrap_decode_seconds this is the tap's own overhead.", + snap.bootstrap_tap_seconds, + ), + ( + "walshadow_bootstrap_channel_block_seconds_total", + "Cumulative time the bootstrap walk spent waiting for a free tuple-channel slot, i.e. emitter drain time seen by the walk.", + snap.bootstrap_channel_block_seconds, + ), + ( + "walshadow_inserter_ch_seconds_total", + "Cumulative inserter time inside the ClickHouse INSERT round trip. Against inserter_pool_size x uptime this is CH-side utilization.", + snap.inserter_ch_seconds_total, + ), + ( + "walshadow_inserter_encode_seconds_total", + "Cumulative inserter time rebuilding the Native block over a batch's owned slabs.", + snap.inserter_encode_seconds_total, + ), + ( + "walshadow_oracle_resolve_seconds_total", + "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", + snap.oracle_resolve_seconds_total, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + writeln!(s, "{name} {secs:.3}").unwrap(); + } + // Process CPU as a float counter (seconds); rate() ≈ cores in use. let name = "walshadow_process_cpu_seconds_total"; writeln!( @@ -1471,6 +1599,37 @@ mod tests { assert!(body.contains("walshadow_route_snapshots_total{result=\"unmapped\"} 8")); } + /// Bootstrap stage attribution reaches `/metrics`. `BootstrapOutcome` + /// counters used to stop at a log line, which is what made a slow + /// initial load unattributable + #[test] + fn render_exposes_bootstrap_stage_attribution() { + let snap = MetricsSnapshot { + bootstrap_bytes_tapped: 1 << 30, + bootstrap_pages_walked: 131_072, + bootstrap_files_skipped_unmapped: 297, + bootstrap_decode_seconds: 12.5, + bootstrap_channel_block_seconds: 3.25, + bootstrap_tap_seconds: 0.0, + inserter_ch_seconds_total: 41.5, + oracle_resolve_seconds_total: 8.25, + ..MetricsSnapshot::default() + }; + let body = render(&snap); + for want in [ + "walshadow_bootstrap_bytes_tapped_total 1073741824", + "walshadow_bootstrap_pages_walked_total 131072", + "walshadow_bootstrap_files_skipped_unmapped_total 297", + "walshadow_bootstrap_decode_seconds_total 12.500", + "walshadow_bootstrap_channel_block_seconds_total 3.250", + "walshadow_bootstrap_tap_seconds_total 0.000", + "walshadow_inserter_ch_seconds_total 41.500", + "walshadow_oracle_resolve_seconds_total 8.250", + ] { + assert!(body.contains(want), "missing {want}"); + } + } + /// Prometheus rejects a family declared twice; `descriptor_ambiguous_total` /// was emitted bare and labelled with contradicting HELP text #[test] diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 298935f3..051f6ab3 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -6,7 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use clickhouse_c::{Allocator, Block, BlockOpts, BlockReader, Column, SliceIo}; use crate::decode::heap_decoder::ColumnValue; -use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES}; +use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES, request_frame}; use crate::schema::RelAttr; /// Cell tags, matching `WS_CELL_*` in `pgext/walshadow.h` @@ -71,6 +71,7 @@ pub struct OracleColumnBuf { pub source_typmod: i32, cells: Vec, wire_bytes: usize, + literal_only: bool, } impl OracleColumnBuf { @@ -80,14 +81,23 @@ impl OracleColumnBuf { source_typmod, cells: Vec::new(), wire_bytes: 0, + literal_only: true, } } pub fn push(&mut self, cell: OracleCell) { + self.literal_only &= matches!(cell, OracleCell::Literal(_) | OracleCell::Default); self.wire_bytes += cell.wire_bytes(); self.cells.push(cell); } + /// No cell needs a source type: each is either bytes the daemon already + /// rendered or a default. Against a `String` target such a column would + /// cross to PG only to be handed straight back + pub fn literal_only(&self) -> bool { + self.literal_only + } + pub fn cells(&self) -> &[OracleCell] { &self.cells } @@ -141,6 +151,12 @@ impl Oracle { } } + /// Requests the shadow answers at once, ie the bridge's pool width. One + /// worker serves one request per loop iteration + pub fn concurrency(&self) -> usize { + self.bridge.pool_size() + } + pub async fn encode_batch( &self, columns: &[OracleRequestColumn<'_>], @@ -158,8 +174,11 @@ impl Oracle { ))); } } - let payload = encode_request(columns, n_rows); - let response = match self.bridge.encode_native(&payload).await { + let response = match self + .bridge + .encode_native(encode_request(columns, n_rows)) + .await + { Ok(r) => r, Err(e) => { if matches!(e, BridgeError::Remote(_)) { @@ -201,13 +220,16 @@ pub const ORACLE_BATCH_SEAL_BYTES: usize = 32 << 20; const _: () = assert!(ORACLE_BATCH_SEAL_BYTES <= MAX_REQUEST_BYTES); -/// Encode all column metadata before row-major cells +/// Encode all column metadata before row-major cells, past the bridge's +/// unwritten frame prefix. Seal bytes reach 32 MiB, so building into the +/// frame rather than into a payload the bridge then copies saves one +/// allocation and one memcpy of the whole request fn encode_request(columns: &[OracleRequestColumn<'_>], n_rows: usize) -> Vec { let size: usize = columns .iter() .map(|c| request_column_bytes(c.name, c.target_type) + c.buf.approx_size()) .sum(); - let mut out = Vec::with_capacity(8 + size); + let mut out = request_frame(8 + size); out.extend_from_slice(&(n_rows as u32).to_be_bytes()); out.extend_from_slice(&(columns.len() as u32).to_be_bytes()); for c in columns { @@ -364,6 +386,7 @@ impl OracleStats { #[cfg(test)] mod tests { use super::*; + use crate::ops::bridge::FRAME_PREFIX_BYTES; fn col<'a>( ordinal: u32, @@ -390,7 +413,7 @@ mod tests { let cols = [col(0, "t", "Array(Int32)", &a), col(3, "j", "JSON", &b)]; let out = encode_request(&cols, 2); - let mut c = 0; + let mut c = FRAME_PREFIX_BYTES; let u32_at = |c: &mut usize| { let v = u32::from_be_bytes(out[*c..*c + 4].try_into().unwrap()); *c += 4; @@ -439,7 +462,7 @@ mod tests { buf.push(OracleCell::DiskRaw(vec![1, 2, 3])); let cols = [col(7, "payload", "Nullable(JSON)", &buf)]; assert_eq!( - encode_request(&cols, 2).len() + 1, // opcode the bridge prepends + encode_request(&cols, 2).len() + 1 - FRAME_PREFIX_BYTES, REQUEST_FRAME_BYTES + request_column_bytes("payload", "Nullable(JSON)") + buf.approx_size() diff --git a/tests/bootstrap_direct_e2e.rs b/tests/bootstrap_direct_e2e.rs index ec8f44b4..98634d25 100644 --- a/tests/bootstrap_direct_e2e.rs +++ b/tests/bootstrap_direct_e2e.rs @@ -181,12 +181,23 @@ async fn direct_source_self_hosted_via_replication_protocol() { // DiskLanderSink coverage assert!( - outcome.disk.kept_files > 100, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed) + > 100, "expected >100 catalog files landed, got {}", - outcome.disk.kept_files, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed), ); assert!( - outcome.disk.skipped_denylist > 0, + outcome + .disk + .skipped_denylist + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no denylist files skipped — expected at least one under pg_replslot/ or pg_stat_tmp/", ); @@ -214,15 +225,46 @@ async fn direct_source_self_hosted_via_replication_protocol() { ); // PageWalkSink stats - assert!(outcome.page_walk.files_seen > 0); - assert!(outcome.page_walk.files_walked > 0); - assert!(outcome.page_walk.pages_walked > 0); assert!( - outcome.page_walk.tuples_emitted >= N_ROWS as u64, + outcome + .page_walk + .files_seen + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .files_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .pages_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) + >= N_ROWS as u64, "expected >= {N_ROWS} tuples emitted from page walk, got {}", - outcome.page_walk.tuples_emitted, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), + ); + assert_eq!( + shipped, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) ); - assert_eq!(shipped, outcome.page_walk.tuples_emitted); // CommittedTuple shape let mut int_ids: HashSet = HashSet::new(); diff --git a/tests/bootstrap_object_store_e2e.rs b/tests/bootstrap_object_store_e2e.rs index 516d1ad0..fa23a03f 100644 --- a/tests/bootstrap_object_store_e2e.rs +++ b/tests/bootstrap_object_store_e2e.rs @@ -284,14 +284,25 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // base// + global/. Use a generous lower bound rather than // a tight count: PG version drift changes the exact number. assert!( - outcome.disk.kept_files > 100, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed) + > 100, "expected >100 catalog files landed, got {}; \ tuned for PG initdb expectations — bump or investigate \ a regression", - outcome.disk.kept_files, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed), ); assert!( - outcome.disk.skipped_denylist > 0, + outcome + .disk + .skipped_denylist + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no denylist files skipped — expected at least one under pg_replslot/ or pg_stat_tmp/", ); @@ -325,23 +336,49 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // --- PageWalkSink stats --- assert!( - outcome.page_walk.files_seen > 0, + outcome + .page_walk + .files_seen + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no user-heap files observed by the tap", ); assert!( - outcome.page_walk.files_walked > 0, + outcome + .page_walk + .files_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no user-heap files walked (filenode mismatch with CatalogMap?)", ); - assert!(outcome.page_walk.pages_walked > 0, "no heap pages walked",); assert!( - outcome.page_walk.tuples_emitted >= N_ROWS as u64, + outcome + .page_walk + .pages_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0, + "no heap pages walked", + ); + assert!( + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) + >= N_ROWS as u64, "expected >= {N_ROWS} tuples emitted from page walk, got {}", - outcome.page_walk.tuples_emitted, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), ); // --- Drain delivered every tuple --- assert_eq!( - shipped, outcome.page_walk.tuples_emitted, + shipped, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), "drain count != page walk emit count — channel dropped tuples" ); diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 465622b8..73a82467 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -160,12 +160,12 @@ async fn bootstrap_tail_fans_out_n2() { // Feed all foo rows then all baz rows — contiguous per rfn, as // PageWalkSink emits. Two seqs, each spanning ~8 budget-sized batches. // cap > 2*ROWS_PER_TABLE so the pre-send fits before the drain spawns - let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::(128); + let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::>(128); for id in 0..ROWS_PER_TABLE { - tup_tx.send(tuple(16400, id)).await.unwrap(); + tup_tx.send(vec![tuple(16400, id)]).await.unwrap(); } for id in 0..ROWS_PER_TABLE { - tup_tx.send(tuple(16401, id)).await.unwrap(); + tup_tx.send(vec![tuple(16401, id)]).await.unwrap(); } drop(tup_tx); diff --git a/tests/bridge.rs b/tests/bridge.rs index 1de4b394..02401448 100644 --- a/tests/bridge.rs +++ b/tests/bridge.rs @@ -20,7 +20,7 @@ use walshadow::bridge::{ }; use walshadow::oracle::{Oracle, OracleCell, OracleColumnBuf, OracleRequestColumn}; use walshadow::pg::socket_conninfo; -use walshadow::schema::ReplIdent; +use walshadow::schema::{NUMERICOID, ReplIdent}; use walshadow::shadow::{BridgeConf, Shadow, ShadowConfig}; use walshadow::shadow_catalog::{CatalogError, ShadowCatalog, ShadowCatalogConfig}; @@ -56,6 +56,10 @@ impl Drop for StopOnDrop { } fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { + start_pg_with_workers(tmp, port, 1) +} + +fn start_pg_with_workers(tmp: &tempfile::TempDir, port: u16, workers: usize) -> StopOnDrop { let lib_dir = pgext_dir(); let mut cfg = ShadowConfig::new(tmp.path().join("data"), tmp.path().join("filtered")); cfg.port = port; @@ -63,6 +67,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { cfg.ctl_timeout = Duration::from_secs(60); let mut bridge = BridgeConf::in_dir(&cfg.socket_dir); bridge.library_dir = Some(lib_dir); + bridge.workers = workers; cfg.bridge = Some(bridge); fs::create_dir_all(&cfg.filter_out_dir).unwrap(); fs::create_dir_all(&cfg.socket_dir).unwrap(); @@ -76,7 +81,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { async fn dial(sh: &Shadow) -> Bridge { let path = sh.bridge_socket().expect("bridge configured"); - walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())) } @@ -115,7 +120,7 @@ async fn open_catalog(sh: &Shadow, bridge: Arc) -> ShadowCatalog { async fn open_mirror_catalog(sh: &Shadow, sock: &Path) -> (Arc, ShadowCatalog) { spawn_moving_worker(tokio::net::UnixListener::bind(sock).expect("bind stand-in")); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(sock, Duration::from_secs(5)) + walshadow::bridge::connect_with_budget(sock, 1, Duration::from_secs(5)) .await .expect("stand-in bridge"), ); @@ -1027,3 +1032,126 @@ async fn bridge_committed_read_falls_back_when_replay_moves() { "{err:?}" ); } + +/// `walshadow.bridge_workers = 4` registers four workers on +/// `socket_path`, `socket_path.1`, `.2`, `.3`. Pooling them is what stops +/// oracle throughput being one backend's conversion rate. +/// +/// Both bridges run against the same shadow, so the comparison holds the +/// worker's conversion rate fixed and varies only the number of sockets the +/// daemon spreads over. A one-socket bridge queues every caller behind the +/// one in front; a four-socket bridge does not. +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn bridge_worker_pool_serves_concurrent_requests() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + const WORKERS: usize = 4; + let tmp = tempfile::tempdir().unwrap(); + let guard = start_pg_with_workers(&tmp, ports::PG_SHADOW_PORT, WORKERS); + let path = guard.sh.bridge_socket().expect("bridge configured"); + + let pooled = Arc::new( + walshadow::bridge::connect_with_budget(path, WORKERS, Duration::from_secs(30)) + .await + .expect("pooled bridge connect"), + ); + // Same shadow, one socket: what every deployment had before the pool + let single = Arc::new( + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(30)) + .await + .expect("single bridge connect"), + ); + assert_eq!(pooled.pool_size(), WORKERS, "one socket per worker"); + assert_eq!(single.pool_size(), 1); + // Every slot dialled its own HELLO; a mismatch would have failed connect + assert!(pooled.info().is_some()); + + // Conversion cost is per cell, so size the batch until one round trip is + // measurable rather than looking for a sleep hook the op does not have. + // Cells are built once and shared: what the pool widens is worker-side + // conversion, so daemon-side request building must stay out of the clock + const ROWS: usize = 100_000; + let cells = Arc::new({ + let mut b = OracleColumnBuf::new(NUMERICOID, -1); + for _ in 0..ROWS { + b.push(OracleCell::DiskRaw(numeric_42())); + } + b + }); + + /// One `ENCODE_NATIVE` per task, all in flight at once + async fn race(bridge: &Arc, cells: &Arc, tasks: usize) -> Duration { + let started = Instant::now(); + let mut set = Vec::new(); + for _ in 0..tasks { + let bridge = bridge.clone(); + let cells = cells.clone(); + set.push(tokio::spawn(async move { + let columns = [OracleRequestColumn { + ordinal: 0, + name: "c0", + target_type: "String", + buf: &cells, + }]; + Oracle::new(bridge) + .encode_batch( + &columns, + cells.cells().len(), + clickhouse_c::Allocator::stdlib(), + ) + .await + .expect("oracle answers") + .column(0) + .and_then(|c| c.string()) + .expect("string column") + .0 + .len() + })); + } + for h in set { + assert_eq!(h.await.unwrap(), ROWS, "one offset per row"); + } + started.elapsed() + } + + // Warm both: first touch pays page faults and the worker's first plan + race(&pooled, &cells, WORKERS).await; + race(&single, &cells, WORKERS).await; + + // Best of three a side, interleaved so a neighbour test's load lands on + // both rather than on whichever ran while the machine was busy + let mut pooled_secs = Duration::MAX; + let mut single_secs = Duration::MAX; + for _ in 0..3 { + pooled_secs = pooled_secs.min(race(&pooled, &cells, WORKERS).await); + single_secs = single_secs.min(race(&single, &cells, WORKERS).await); + } + let speedup = single_secs.as_secs_f64() / pooled_secs.as_secs_f64(); + eprintln!( + "bridge pool: {WORKERS} concurrent ENCODE_NATIVE of {ROWS} cells — \ + {WORKERS} sockets {pooled_secs:?}, 1 socket {single_secs:?} ({speedup:.2}x)", + ); + + // Cores cap the win, not pool width: four workers plus the daemon's own + // decode share a 4-vCPU CI runner, which lands ~1.7x where a 16-core box + // reaches ~2.4x. A funnel scores 1.0x, so the bar sits between + assert!( + speedup > 1.4, + "pool bought nothing: {pooled_secs:?} pooled vs {single_secs:?} on one socket", + ); + // Pool width is operator-visible + assert_eq!( + pooled + .stats + .pool_size + .load(std::sync::atomic::Ordering::Relaxed), + WORKERS as u64, + ); + // Catalog reads stayed on worker 0 whatever the pool width + pooled + .scan(Catalog::Namespace, 0, &[]) + .await + .expect("scan pinned to slot 0 still answers"); +} diff --git a/tests/common/inproc_harness.rs b/tests/common/inproc_harness.rs index 5721f683..d5ebc3dd 100644 --- a/tests/common/inproc_harness.rs +++ b/tests/common/inproc_harness.rs @@ -725,7 +725,7 @@ async fn build_pipeline_inner( }; let bridge_path = shadow.bridge_socket().expect("bridge configured"); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(bridge_path, Duration::from_secs(60)) + walshadow::bridge::connect_with_budget(bridge_path, 1, Duration::from_secs(60)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", bridge_path.display())), ); diff --git a/tests/oracle.rs b/tests/oracle.rs index e9fdb9ec..7c11cb0a 100644 --- a/tests/oracle.rs +++ b/tests/oracle.rs @@ -77,7 +77,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> Option { async fn oracle_on(sh: &Shadow) -> Oracle { let path = sh.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + let bridge = walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())); Oracle::new(Arc::new(bridge)) @@ -315,12 +315,14 @@ async fn worker_refuses_malformed_requests() { return; }; let socket = guard.sh.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(20)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(20)) .await .expect("bridge connect"); + // Past the bridge's frame prefix, as `encode_request` builds it let framed = |rows: u32, cols: u32, meta: &[u8], cells: &[u8]| -> Vec { - let mut v = rows.to_be_bytes().to_vec(); + let mut v = walshadow::bridge::request_frame(0); + v.extend_from_slice(&rows.to_be_bytes()); v.extend_from_slice(&cols.to_be_bytes()); v.extend_from_slice(meta); v.extend_from_slice(cells); @@ -363,7 +365,7 @@ async fn worker_refuses_malformed_requests() { ]; for (what, payload) in cases { let err = bridge - .encode_native(&payload) + .encode_native(payload) .await .err() .unwrap_or_else(|| panic!("{what}: worker accepted a malformed request")); @@ -376,7 +378,7 @@ async fn worker_refuses_malformed_requests() { // Reject bytes beyond declared cells let mut trailing = framed(1, 1, &one, &[0x01, 0, 0, 0, 4, 42, 0, 0, 0]); trailing.push(0); - assert!(bridge.encode_native(&trailing).await.is_err()); + assert!(bridge.encode_native(trailing).await.is_err()); // Parser errors preserve connection let oracle = Oracle::new(Arc::new(bridge)); diff --git a/tests/oracle_types_e2e.rs b/tests/oracle_types_e2e.rs index bf5cde9e..11f10739 100644 --- a/tests/oracle_types_e2e.rs +++ b/tests/oracle_types_e2e.rs @@ -101,7 +101,7 @@ async fn run_oracle_stats( // Worker binds only once recovery reaches consistency, so budget the dial let socket = shadow.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(30)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) .await .expect("bridge connect"); assert!( diff --git a/tests/runtime_config_e2e.rs b/tests/runtime_config_e2e.rs index 4e534ae3..7bdb0a90 100644 --- a/tests/runtime_config_e2e.rs +++ b/tests/runtime_config_e2e.rs @@ -478,7 +478,7 @@ async fn opt_in_non_empty_backfills_pre_opt_in_rows() { // `meta jsonb` sits outside the local matrix, so the backfill tail needs // the same oracle the WAL path uses let socket = shadow.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(30)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) .await .expect("bridge connect"); let oracle = Arc::new(walshadow::oracle::Oracle::new(Arc::new(bridge))); diff --git a/tests/shadow_catalog.rs b/tests/shadow_catalog.rs index 37bade07..e1b4909f 100644 --- a/tests/shadow_catalog.rs +++ b/tests/shadow_catalog.rs @@ -81,7 +81,7 @@ async fn open_catalog(shadow: &Shadow, replay_timeout: Duration) -> ShadowCatalo async fn open_bridge(shadow: &Shadow) -> Arc { let path = shadow.bridge_socket().expect("bridge configured"); Arc::new( - walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())), ) diff --git a/tests/xact_buffer.rs b/tests/xact_buffer.rs index 7a05f936..1518ca9d 100644 --- a/tests/xact_buffer.rs +++ b/tests/xact_buffer.rs @@ -95,6 +95,7 @@ async fn open_catalog(shadow: &Shadow) -> ShadowCatalog { let bridge = Arc::new( walshadow::bridge::connect_with_budget( shadow.bridge_socket().expect("bridge configured"), + 1, Duration::from_secs(20), ) .await From fb734fa86f5fff3780c4d983137acfc8b9969cdf Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:50:41 +0000 Subject: [PATCH 2/2] remove perf check from ci, add plans/future/perf_regression.md, get rid of plans/future/parked.md --- plans/decoder.md | 2 +- plans/future/INDEX.md | 3 +- plans/future/coverage100.md | 6 + plans/future/ddl_fuzz.md | 2 + plans/future/parked.md | 66 +------ plans/future/perf_regression.md | 245 +++++++++++++++++++++++++ plans/future/risks.md | 30 ++- plans/future/runtime_config_from_pg.md | 1 + tests/bridge.rs | 46 ++--- 9 files changed, 303 insertions(+), 98 deletions(-) create mode 100644 plans/future/perf_regression.md diff --git a/plans/decoder.md b/plans/decoder.md index 3e32e86f..62a7fe92 100644 --- a/plans/decoder.md +++ b/plans/decoder.md @@ -373,7 +373,7 @@ captured fixtures across PG 16/17/18. `tests/classify_fixture.rs` infra snapshots MULTI_INSERT fixtures; the same fixture gap covers `XACT_XINFO_HAS_SUBXACTS` layout. Drift in either record's per-major shape would surface as silent decode mismatch on exactly one major. -Tracked in [future/parked.md](future/parked.md). Structural cousins — +Tracked in [future/risks.md](future/risks.md). Structural cousins — `xl_heap_*` headers, `xl_heap_multi_insert` field order, `attmissingval` encoding — are stable per WAL alignment memory note + direct catalog cross-check against PG 16/17/18 source, absent diff --git a/plans/future/INDEX.md b/plans/future/INDEX.md index 46797e30..f1e55350 100644 --- a/plans/future/INDEX.md +++ b/plans/future/INDEX.md @@ -20,7 +20,8 @@ rationale under `plans/` only when code cannot express it * [pinned_ddl_baseline.md](pinned_ddl_baseline.md) — schema-event outcome must be a function of config + baseline, not cache warmth: CH-existence / persisted-baseline options for cross-restart consistency, drop detection across downtime, opt-in mapping vs republish * [coverage100.md](coverage100.md) — drive `cargo llvm-cov` line coverage toward 100%: tiered work list (pure units → fixtures → live e2e → hard tail) * [FUZZ.md](FUZZ.md) — continuous coverage-guided fuzzing (cargo-fuzz/libFuzzer) across wal-rus + walshadow + clickhouse-c-rs: tiered targets, round-trip/differential oracles, C-boundary ASan, unattended-VM supervisor +* [perf_regression.md](perf_regression.md) — performance measurement off CI machines: no wall-clock assertions in the suite, own workflow on dedicated hardware, `bench/` initial-load shape, disposable `benches/` microbenches, regression bands * [pipeline_backpressure_and_scaling.md](pipeline_backpressure_and_scaling.md) — parallel decode+insert pipeline: WAL-pump backpressure via wire/record split, decode/insert scaling (bootstrap Option B, hot-table sharding, N/M sizing); pipeline substrate in [emitter.md](../emitter.md) * [dependencies.md](dependencies.md) — crates.io replacement candidates for generic object storage, MPMC, retry, throttling, and metrics code * [risks.md](risks.md) — measurement-deferred risks and open questions -* [parked.md](parked.md) — small operational polish + cross-major fixtures + skipped-test drive +* [parked.md](parked.md) — small operational polish + walsender hardening (TLS/SCRAM, `hot_standby_feedback`) diff --git a/plans/future/coverage100.md b/plans/future/coverage100.md index 00776da1..f0b62200 100644 --- a/plans/future/coverage100.md +++ b/plans/future/coverage100.md @@ -97,6 +97,12 @@ Use existing in-proc and fixture paths. Extend Extend existing `WALSHADOW_USE_LOCAL` e2e tests. Keep live-state cases in existing harnesses rather than adding binaries +Skip gates here are runtime, not `#[ignore]`: a suite checks for `initdb` / +`pg_basebackup` / `clickhouse` on PATH and returns early when absent, so a +job missing one reports the suite as passing while its lines stay uncovered +for no visible reason. Read durations, a live suite that "passes" in ~0 s +did not run + - `shadow_catalog.rs::seed_from_source` and dependent closures: drive via `tests/catalog_seed.rs` / `tests/shadow_catalog.rs` - `ch_emitter.rs::reconnect`: explicit CH drop mid-stream diff --git a/plans/future/ddl_fuzz.md b/plans/future/ddl_fuzz.md index d0738d26..4fe67f1a 100644 --- a/plans/future/ddl_fuzz.md +++ b/plans/future/ddl_fuzz.md @@ -55,6 +55,8 @@ limits from new interactions and gives shrinker stable endpoints | drop PK constraint, drop its column | CH still uses column in `ORDER BY`; CH DROP COLUMN fails after any earlier xact effects | preflight whole plan against target key | | add PK to table created keyless | CH retains `ORDER BY (_lsn)`; updates do not collapse by new PK | rebuild key or keep case unsupported | | `DROP TABLE; CREATE TABLE` with same name under each drop strategy | `retain` / `warn` preserve old rows and `CREATE IF NOT EXISTS` no-ops; `drop` should round-trip | encode policy-specific lifecycle oracle | +| `DROP TABLE s.t CASCADE` with a dependent view, FK child or partition child | basic path emits the drop via `SchemaEvent` + `DrainEntry::Catalog`; dependent relations get no event of their own | decide whether cascaded relations follow the drop strategy or are left to drift | +| `DROP TABLE s.t RESTRICT` refused by source, then DML on `s.t` | source rejects, so no WAL and no event; mapping must stay intact | assert a refused DDL leaves no destination effect | | `CREATE UNLOGGED TABLE; INSERT ...` | catalog may create CH table; user DML has no durable WAL | reject mapping or mark no-row policy explicitly | | `ALTER TABLE ... SET UNLOGGED`, mutate, `SET LOGGED` | unlogged interval disappears; stale CH rows can survive conversion | reject persistence transition | | attach populated partition, then write through parent | heap WAL names leaf; pinned parent target receives no fan-in and attach emits no row backfill | define partition routing/backfill semantics | diff --git a/plans/future/parked.md b/plans/future/parked.md index b9e01633..7ab70514 100644 --- a/plans/future/parked.md +++ b/plans/future/parked.md @@ -1,7 +1,12 @@ # parked — small operational debt and follow-up polish -Operational debt collected from retros plus follow-ups from the -allocation-audit pass. One-line per item +Operational debt collected from retros, one line per item. Items that fit +a subject doc live there instead: perf and profiling work in +[perf_regression.md](perf_regression.md), coverage and skip-gate hazards in +[coverage100.md](coverage100.md), decoder-fidelity risks in +[risks.md](risks.md), config knobs in +[runtime_config_from_pg.md](runtime_config_from_pg.md), DDL transition +corpus in [ddl_fuzz.md](ddl_fuzz.md) ## v1.0 operational polish @@ -13,49 +18,6 @@ allocation-audit pass. One-line per item spawn own CH (~5 s × N startup). Total CI cost ~25 s of unique boot time. Flag if test count doubles -## Cross-major fixture pinning - -* **MULTI_INSERT + xl_xact_commit fixtures against PG 16/17/18 via - `tests/classify_fixture.rs`.** Cross-major drift in tail-walk - semantics would surface as silent decoder mismatch under one - specific major. cross-major snapshot fixtures called for snapshot - fixtures across majors - -## Drive currently-skipped tests - -Acceptance tests ship with runtime skip-gates checking for `initdb` / -`pg_basebackup` / `clickhouse` on `PATH`; *not* `#[ignore]`. Each -needs source PG + CH + (usually) basebackup-cloned shadow. Drive in -CI when those binaries reliably present: - -* `kill_restart` -* `pgbench_acceptance` -* `bootstrap_direct_ch` -* `bootstrap_object_store_ch` -* `truncate` -* `subxact` -* `copy_into` -* `add_column_default` - -Each is a one-line un-skip + observation of which fixture path -needs a kick. Acceptance items §1 (pgbench), §5 (kill-restart) -remain unverified against live topology until driven - -## Zero-copy follow-ups - -* **Criterion benchmark.** Allocation-count + RSS measurement - post-hoc; land `benches/` crate when measurement contested. - Targets predicted RSS drop (≈200 MB → ≈0 for 100k-record - heap-INSERT segment) + 1.5-3× decode throughput from dropped - allocator pressure -* **`XLogRecord.blocks` smallvec.** Records average 0-2 blocks; - `SmallVec<[_; 2]>` keeps common case stack-resident. Allocation - polish below byte-traffic wins from Cow -* **Header-walk single-pass merge.** `record.blocks` walk runs - twice (once for IDs, once for payloads) in wal-rus parser. Merge - into single pass since IDs arrive in order. Leftover from wal-g - port - ## Walsender hardening * **TLS / SCRAM auth.** Trust-over-loopback only today. @@ -68,17 +30,3 @@ remain unverified against live topology until driven * **Walsender keepalive-timeout unit test.** Indirectly covered by libpq + PG-walreceiver round-trips in `walsender_pg18_walreceiver`; explicit unit test is polish - -## Decoder follow-ups - -* **Subxact `XACT_XINFO_HAS_INVALS` ordering verification fixture.** - Capture commit record from PG with all `xinfo` bits set; prove - walk doesn't drift under out-of-the-way ordering on some major. - Subxact retro flagged this -* **TRUNCATE strategy knob.** v1 emits single `TRUNCATE TABLE ` - per relation. Per-table `truncate_strategy = "passthrough" | - "ignore"` knob once downstream consumer asks. Defer-until-asked -* **DROP TABLE propagation polish.** Basic path runs via - `SchemaEvent` + `DrainEntry::Catalog` channel. Corner cases - (CASCADE, RESTRICT, dependent objects) need pinning against - fixture matrix diff --git a/plans/future/perf_regression.md b/plans/future/perf_regression.md new file mode 100644 index 00000000..3599b87d --- /dev/null +++ b/plans/future/perf_regression.md @@ -0,0 +1,245 @@ +# perf_regression — measurement off CI machines + +Performance claims get their own workflow on hardware nobody else is +using. `cargo nextest` (and the coverage job that wraps it) proves +behavior; it never proves speed. This doc sets that boundary, then +specifies the workload `bench/` needs to cover initial load, which is the +largest unmeasured stage + +## Why CI cannot hold a ratio + +A GitHub-hosted runner is 4 shared vCPUs running the whole suite at +`num-cpus` width, so a timing assertion measures scheduler luck. The +coverage job is worse: instrumentation adds counter writes to every +region, which lands unevenly across a pooled path and moves the ratio +with no code change + +Concretely, the bridge worker pool compares 4 concurrent `ENCODE_NATIVE` +over 4 sockets against the same 4 over 1 socket, best of three a side, and +asserts speedup above 1.4. It scores ~2.4x on a 16-core box, ~1.7x on a +4-vCPU runner and 1.24x (90.9ms vs 112.6ms) under `cargo llvm-cov` on a +runner also executing ~1200 other tests. Cores cap the win, so the bar can +only be set for a machine that the suite does not get to pick + +Rule: no assertion in the test suite may compare wall clock against a +constant or against another wall clock. Printing timings is fine, the WAL +pump microbenchmark test does exactly that and asserts only record counts + +## What the suite keeps + +Splitting a perf test means keeping its correctness half, which is most of +it. For the bridge pool that is: pool width equals the requested worker +count and is visible in `stats.pool_size`, every slot completed its own +HELLO, `info()` is populated, each concurrent request returns one offset +per row, and catalog `scan` stays pinned to slot 0 whatever the width. +None of those touch a clock + +The ratio moves to the oracle roundtrip stage, whose job already is +pricing one batch alone and with the pool saturated. Report core count +beside the ratio, since that is the term that sets the ceiling + +## bench/ is durable, benches/ is not + +Two homes, different lifetimes: + +* `bench/` is a deployed workload engine: a crate with shapes, an + instrument split (`count_all` cadence for throughput, `count_id` probes + for latency), a `Destination` abstraction covering ClickHouse and a PG + standby, plus the EC2 harness that stands up a comparable box. Shapes + are cross-engine, so they survive any internal refactor +* `benches/*.rs` are task-scoped microbenches. Each exists to settle one + contested optimisation (pump-vs-drain CPU split, oracle framing passes) + and stops earning its keep once that question closes. Expect the + directory to churn and to empty + +So the perf workflow keys off `bench/` shapes and never off a `[[bench]]` +target name. Deleting a microbench must stay a file delete plus a manifest +stanza: `docker/Dockerfile.bench` already copies `src` and `bench` without +`benches` and builds `-p walshadow-bench`, so the deployed driver does not +see them. A microbench that does stay must answer `--list` with +`: benchmark`, else `nextest --all-targets` fails listing + +## Finding the next bottleneck + +Gating answers "did this get worse". Optimisation work asks the other +question, "what owns the time", and that needs a profiler plus one +microbench per contested claim, on the same dedicated box + +On-CPU capture: `bench/ec2/profile.sh [secs]` starts a capture +beside a run and teardown copies the result into the node folder. The +release profile carries `debug = "line-tables-only"`, so `perf annotate` +and `--sort=srcline` attribute samples to source lines at no runtime cost + +Criterion earns its keep where a claim is a per-call cost over a stable +input: statistical comparison, saved baselines, outlier detection. Where +the claim is a staged pipeline needing allocator choice, thread counts and +synthetic segments, the flag-driven `harness = false` shape stays. Either +way the allocator must match the daemon's (mimalloc): the walk allocates +decoded values on one thread and the drain frees them on another, and +glibc's shared arena serializes on that pattern hard enough to invert the +conclusion, reporting a parallel pump as slower than the serial one + +Allocation count and peak RSS are a separate instrument from wall clock, +answered post hoc by a counting allocator wrapper plus RSS sampling, per +stage. The zero-copy framing pass predicts a 100k-record heap-INSERT +segment dropping ~200 MB of allocator traffic to ~0, worth 1.5-3x decode +throughput off lost allocator pressure. Unmeasured + +Candidates already named, each wanting a number before code: + +* **CRC32C rewrite parallelism.** The filter rewrites every kept record's + CRC on one thread at ~1 ns/byte, so 1 GB/s of WAL saturates a core. + Records are independent post-classification, see [risks.md](risks.md) +* **`XLogRecord.blocks` allocation.** `Vec` per record + where records average 0-2 blocks; `SmallVec<[_; 2]>` keeps the common + case stack-resident. Ranked below byte-traffic wins +* **Block-header walk.** The parser walks blocks twice, once pushing + headers and once attaching image/data slices. Not redundant: a record + body carries every block header before any block payload, which is why + PostgreSQL's own `DecodeXLogRecord` splits the same way. Any win here is + pre-sizing the vector, not merging the passes + +## Workflow shape + +Trigger on `workflow_dispatch` plus a schedule, never on pull_request. +Runs either on a single-tenant self-hosted runner or by driving +`bench/ec2/stack.sh` (`up `, `bench run `, `down`), which +already pins instance type and AZ and writes `provenance.txt` (setup, +instance type, AZ, commit) beside the results. Publish numbers as +artifacts; do not fail a PR on them + +## Shape: initial load + +`bench/` covers steady-state only. Initial load is where BASE_BACKUP wire +throughput, source-PG read cost and ClickHouse insert saturation meet, and +it has no shape + +### Two triggers, different cost + +Greenfield bootstrap (`--bootstrap-mode=direct|object_store`) runs only +against an empty shadow data dir with no completion marker. Triggering it +means wiping the data volume and restarting the container, lifecycle +control the bench engine does not have by design + +Per-table backup backfill (`initial_load='base_backup'|'object_store'`) +drives the same `PageWalkSink` → gate → bootstrap-drain path on a live +daemon. Trigger is an INSERT into `config_table` on the source plus +`pg_switch_wal()`, nothing else + +Both cover the page walk, the backup sink and source, and the bootstrap +drain. Per-table needs no restart and no SSH, so it carries the primary +workload; greenfield rides a second shape behind a reset hook + +### Assumptions the engine breaks + +* `run()` truncates the source table unconditionally. An initial-load + dataset *is* pre-existing source state, so the clean-slate preamble goes + per shape: clear the destination, never the source +* `Throughput::from_curve`'s `max_backlog` term is `achieved_rate * at`, + meaningful only for an interval-driven producer. Against a static source + it is fabricated, so suppress it. `all_visible_at` and `peak_rate` stay + valid +* The latency instrument has no meaning here, no commit instant exists. + `count_id` probes, probe slot states and the probe clock all drop out. + Throughput instrument only +* Seed into `demo.bl_`, never the table other shapes truncate. An + auto-create namespace rule already covers new tables there + +### `--bench initial-load` + +1. **Seed** `CREATE TABLE demo.bl_`, fill server-side via + `INSERT … SELECT FROM generate_series`, rows and width from flags. + Report seed wall time and `pg_total_relation_size` apart from the + measurement. Relation size is the MiB/s denominator +2. **Trigger** INSERT into `config_table(namespace, relname, replicate, + initial_load)`, then `pg_switch_wal()`. `t0` is that commit on the + driver's own monotonic clock, no cross-host coordination +3. **Watch** reuse the `count_all` sampler at `count_interval_ms`. + Completion predicate `count_all >= N` is exact while the source stays + static, no FINAL, matching the no-FINAL discipline elsewhere +4. **Report** elapsed, rows/s, MiB/s over heap bytes, peak sampled rate, + plus stage attribution when a metrics endpoint is configured + +Per-run reset is a fresh table name, not a restart: the spill-dir backfill +ledger marks a qname done and no later boot re-runs it. A fresh table per +run also grows cluster size, which is how `base_backup`'s cluster-sized +bandwidth cost becomes visible + +Mode sweep (`copy` vs `base_backup` vs `object_store`) is one flag on the +same shape, under one instrument + +### Concurrent-write variant + +`--concurrent-rate` inserts during a pass, exercising walk/live-stream +overlap and the staging swap. The completion predicate must then become +`count(distinct id) == N` or FINAL: overshoot past N is legitimate there, +dedup absorbs it at merge + +### Stage attribution + +`--metrics-url` scrapes the daemon's metrics endpoint on the count cadence +(GET; the existing ClickHouse HTTP client is POST-only, so this needs a +sibling): + +* `walshadow_config_backfills_pending{mode="…"}`, the daemon-side + start/stop bracket for a pass +* `walshadow_bootstrap_decode_seconds_total`, + `walshadow_bootstrap_tap_seconds_total`, + `walshadow_bootstrap_channel_block_seconds_total` +* `walshadow_inserter_ch_seconds_total`, + `walshadow_inserter_encode_seconds_total` +* `walshadow_process_cpu_seconds_total` + +Answers which of tap, decode, channel block or ClickHouse insert owns wall +clock at real scale. Unset skips it, so the same shape still runs against +PeerDB's snapshot and a standby's `pg_basebackup`, giving a cross-engine +initial-load comparison + +### Daemon prerequisites + +* The per-table backfill builds its walk sink without stats and hands the + source a throwaway `PumpStats`, discarding every stage counter. Thread a + shared progress handle through the pass context and publish it (same + series, or `phase=` labelled) or the restart-free workload stays + wall-clock only +* The bootstrap-phase ticker publishes bootstrap fields over + `MetricsSnapshot::default()`, so every other series, uptime included, + reads 0 for the bootstrap duration then jumps when the status loop takes + over. That reads as a counter reset to any scraper and rules out uptime + as a `t0` anchor. Merge into the live snapshot instead of replacing it + +### `--bench bootstrap` (greenfield) + +Keep out of `--suite`: the steady-state pass is self-contained, this is +not. Add `stack.sh bench bootstrap ` alongside `run`/`fetch`, +sequencing seed → wipe volume + redeploy → one shape → fetch + +Give the shape `--reset-cmd ''` so the bench owns `t0`: run the +command, anchor at the first metrics scrape that answers *after* a +refusal. A first scrape that already answers means daemon start time is +unknown, so report that instead of anchoring wrong, the same rule the +sustained shape applies when rows go missing. One flag covers local +compose and EC2 SSH without the engine knowing either; SSH stays out of +the engine and the local driver shares the flag + +### Relation to the pump microbench + +Not redundant, and not a substitute. The microbench isolates pump-vs-drain +CPU on one host with in-memory tars. This workload measures wire +throughput, source-PG cost and destination insert saturation at scale, and +it outlives the optimisation that motivated the microbench + +## Calling a regression + +Numbers from a shared box are unusable, and numbers from a dedicated box +are only usable with a band. Establish the band before gating anything: +repeat the same shape on the same commit until the spread is known, and +express thresholds in terms of that spread, not a guessed percent + +* Report median and best-of together. Best-of hides contention, median + hides the tail; a move in one only is a machine story +* Require a regression to reproduce on a second pass before it counts +* Key stored baselines by (setup, instance type, shape flags, commit). + A baseline from another instance type is not a baseline +* Publish first, gate later. A perf gate that fires on variance gets + disabled, which is worse than no gate diff --git a/plans/future/risks.md b/plans/future/risks.md index bad7e928..2f5dfc1c 100644 --- a/plans/future/risks.md +++ b/plans/future/risks.md @@ -66,6 +66,32 @@ not from source. Source on tzdata 2023d, shadow's host on tzdata 2024a, divergence on rare-but-real timestamp tzname output. Mitigation: pin tzdata version at deploy time. Not enforced +## Cross-major commit-record tail walk + +A commit record's tail is variable-shape: `xl_xact_commit` appends +subxact array, dropped-stats items, relfilelocators, invalidation +messages, twophase gid and origin, each behind its `xl_xact_xinfo` bit. +Reaching a later field means skipping every earlier one at the right +width, so a wrong width reads a valid-looking record wrong and CRC +catches nothing: the bytes are intact, the interpretation is not + +The shape does move. `xl_xact_stats_item` went 12 → 16 bytes in PG 18 +(PostgreSQL commit `b14e9ce7d55`, `PgStat_HashKey.objid` widened to 8 +bytes carried as two `uint32`), and the `SysCacheIdentifier` values the +invalidation walk matches on shifted 35/36 → 37/38 in the same major. +[`src/decode/wal_xact.rs`](../../src/decode/wal_xact.rs) branches on +`page_magic` for both. `MULTI_INSERT` carries the same class of exposure, +its per-tuple walk reads header flags rather than a fixed stride + +Risk is a per-major branch that no test pins, so the next widening or id +shift is silent on exactly one major. Mitigation is fixture pinning +through [`tests/classify_fixture.rs`](../../tests/classify_fixture.rs): +capture a commit record with every `xinfo` bit set (subxacts, dropped +stats, relfilelocators, `XACT_XINFO_HAS_INVALS`, origin) plus a +`MULTI_INSERT` batch, on 16 / 17 / 18, and assert each major's walk lands +on the same fields. Capture is already scripted per major; the snapshots +are what is missing + ## Path A CRC at >1 GB/s WAL Filter rewrites every kept record's CRC32C; today single-threaded. @@ -77,8 +103,8 @@ classification). Defer thread pool until measurement demands. overview.md pitfall #8 flagged this Zero-copy framing already cut allocator pressure off the hot path; -CRC is the next bottleneck if `criterion` benchmarks land and -surface it. Bench is itself deferred, see [parked.md](parked.md) +CRC is the next bottleneck once a bench surfaces it, see +[perf_regression.md](perf_regression.md) ## PG fork temptation diff --git a/plans/future/runtime_config_from_pg.md b/plans/future/runtime_config_from_pg.md index fcfccec1..dd47bad5 100644 --- a/plans/future/runtime_config_from_pg.md +++ b/plans/future/runtime_config_from_pg.md @@ -157,6 +157,7 @@ behind it, distinct from knobs resolved by [`src/config.rs`](../../src/config.rs |---|---|---|---| | `engine` / `order_by` | table, namespace | text | fixed in `ch_ddl` (engine hardcoded `ReplacingMergeTree`, order_by derived from PK/replica-identity index); shape change, needs rfn drain + `TablePlan` rebuild | | `exclude` | column | bool | `ColumnMapping` has no such field; drops a column from projection + future DDL; shape change | +| `truncate_strategy` | table, namespace | text | `passthrough` (today's single `TRUNCATE TABLE ` per relation) or `ignore`; non-shape key, defer until a downstream consumer asks | | `ch_settings` | global, namespace, table | jsonb | applied to INSERT/CREATE TABLE, merged narrow-wins | | `sample_rate` | (TOML only) | float | emitter row-drop sampling for debug, distinct from the `--validate` oracle sampler in `src/oracle.rs` | | `signal_prefix` | (TOML only) | text | which `pg_logical_emit_message` prefix to scan | diff --git a/tests/bridge.rs b/tests/bridge.rs index 02401448..8afc8f49 100644 --- a/tests/bridge.rs +++ b/tests/bridge.rs @@ -1037,10 +1037,11 @@ async fn bridge_committed_read_falls_back_when_replay_moves() { /// `socket_path`, `socket_path.1`, `.2`, `.3`. Pooling them is what stops /// oracle throughput being one backend's conversion rate. /// -/// Both bridges run against the same shadow, so the comparison holds the -/// worker's conversion rate fixed and varies only the number of sockets the -/// daemon spreads over. A one-socket bridge queues every caller behind the -/// one in front; a four-socket bridge does not. +/// Asserts routing, not rate: four in-flight `ENCODE_NATIVE`s over four +/// sockets each get their own answer back. The pooled-vs-single throughput +/// ratio is hardware-bound (cores cap it) so it is measured by the perf +/// workload, never asserted here, see +/// [`plans/future/perf_regression.md`](../plans/future/perf_regression.md) #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn bridge_worker_pool_serves_concurrent_requests() { if !pg_available() { @@ -1057,7 +1058,7 @@ async fn bridge_worker_pool_serves_concurrent_requests() { .await .expect("pooled bridge connect"), ); - // Same shadow, one socket: what every deployment had before the pool + // Same shadow, one socket: a budget below the worker count is honoured let single = Arc::new( walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(30)) .await @@ -1068,11 +1069,9 @@ async fn bridge_worker_pool_serves_concurrent_requests() { // Every slot dialled its own HELLO; a mismatch would have failed connect assert!(pooled.info().is_some()); - // Conversion cost is per cell, so size the batch until one round trip is - // measurable rather than looking for a sleep hook the op does not have. - // Cells are built once and shared: what the pool widens is worker-side - // conversion, so daemon-side request building must stay out of the clock - const ROWS: usize = 100_000; + // Wide enough that requests are still overlapping when the last one is + // dispatched, which is the state a shared slot would have to serialize + const ROWS: usize = 10_000; let cells = Arc::new({ let mut b = OracleColumnBuf::new(NUMERICOID, -1); for _ in 0..ROWS { @@ -1082,8 +1081,7 @@ async fn bridge_worker_pool_serves_concurrent_requests() { }); /// One `ENCODE_NATIVE` per task, all in flight at once - async fn race(bridge: &Arc, cells: &Arc, tasks: usize) -> Duration { - let started = Instant::now(); + async fn race(bridge: &Arc, cells: &Arc, tasks: usize) { let mut set = Vec::new(); for _ in 0..tasks { let bridge = bridge.clone(); @@ -1113,34 +1111,12 @@ async fn bridge_worker_pool_serves_concurrent_requests() { for h in set { assert_eq!(h.await.unwrap(), ROWS, "one offset per row"); } - started.elapsed() } - // Warm both: first touch pays page faults and the worker's first plan race(&pooled, &cells, WORKERS).await; + // One socket carrying the same concurrency answers every caller too race(&single, &cells, WORKERS).await; - // Best of three a side, interleaved so a neighbour test's load lands on - // both rather than on whichever ran while the machine was busy - let mut pooled_secs = Duration::MAX; - let mut single_secs = Duration::MAX; - for _ in 0..3 { - pooled_secs = pooled_secs.min(race(&pooled, &cells, WORKERS).await); - single_secs = single_secs.min(race(&single, &cells, WORKERS).await); - } - let speedup = single_secs.as_secs_f64() / pooled_secs.as_secs_f64(); - eprintln!( - "bridge pool: {WORKERS} concurrent ENCODE_NATIVE of {ROWS} cells — \ - {WORKERS} sockets {pooled_secs:?}, 1 socket {single_secs:?} ({speedup:.2}x)", - ); - - // Cores cap the win, not pool width: four workers plus the daemon's own - // decode share a 4-vCPU CI runner, which lands ~1.7x where a 16-core box - // reaches ~2.4x. A funnel scores 1.0x, so the bar sits between - assert!( - speedup > 1.4, - "pool bought nothing: {pooled_secs:?} pooled vs {single_secs:?} on one socket", - ); // Pool width is operator-visible assert_eq!( pooled