diff --git a/Cargo.toml b/Cargo.toml index 85eece44..f5ad7c3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,17 @@ opentelemetry-otlp = { version = "0.32", default-features = false, features = [" tracing-opentelemetry = { version = "0.33", default-features = false } mimalloc = "0.1.52" +# `harness = false` throughout: these take shape flags rather than libtest +# filters. Bootstrap pump + drain needs no PG and no CH; oracle batch needs +# a shadow PG for its roundtrip stage only +[[bench]] +name = "bootstrap_pump" +harness = false + +[[bench]] +name = "oracle_batch" +harness = false + [dev-dependencies] tempfile = "3" # Enable paused time for batch deadline tests diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs new file mode 100644 index 00000000..cf837a7a --- /dev/null +++ b/benches/bootstrap_pump.rs @@ -0,0 +1,481 @@ +//! Bootstrap pump + drain throughput, no Postgres and no ClickHouse. +//! +//! Two stages the greenfield initial load is made of, measured apart: +//! +//! - `pump`: in-memory BASE_BACKUP tars → `MultiplexSink(DiskLanderSink, +//! PageWalkSink)` → tuple channel → null drain. Covers tar parse, page +//! framing, heap decode and the tuple-channel hop, under `--parallelism` +//! concurrent tar parts so shared-sink contention shows up. +//! - `drain`: synthetic `BackfillTuple`s → `pipeline::bootstrap::drain` → +//! metrics-only tail. Covers routing plus the `BatcherMsg` hop. +//! +//! ```text +//! cargo bench --bench bootstrap_pump -- \ +//! --stage pump --parts 8 --segments-per-part 4 --pages 1024 --parallelism 4 +//! ``` +//! +//! Must run on mimalloc, the allocator `walshadow-stream` sets: the walk +//! produces decoded values on one thread and the drain frees them on +//! another, and glibc's shared arena serializes on that pattern hard +//! enough to invert the conclusion — there the parallel pump measures +//! slower than the serial one it replaced. +//! +//! `harness = false`, so nextest lists it by answering `--list`. + +// Matches `walshadow-stream`: the walk allocates decoded values on one +// thread and the drain frees them on another, and glibc's shared arena +// serializes on that pattern. A bench on a different allocator measures a +// different program +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use walrus::pg::replication::base_backup::Tablespace; +use walrus::pg::walparser::RelFileNode; + +use walshadow::backfill::backup_source::{ + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, +}; +use walshadow::backfill_bootstrap::{BootstrapConfig, spawn_greenfield_bootstrap}; +use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, PAGE_BYTES}; +use walshadow::heap_decoder::ColumnValue; +use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; +use walshadow::pipeline::bootstrap; +use walshadow::pos::{EmitterAck, Monotone}; +use walshadow::schema::{INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent}; +use walshadow::toast::ToastResolver; + +const DB_NODE: u32 = 5; +const FIRST_FILENODE: u32 = 16400; +const START_LSN: u64 = 0x5000_0000; +/// `HeapTupleHeaderData` + pad to the 8-byte-aligned user-data offset +const TUPLE_HEADER: usize = 24; +const SIZE_OF_PAGE_HEADER: usize = 24; +const SIZE_OF_ITEM_ID: usize = 4; +const LP_NORMAL: u32 = 1; + +#[derive(Clone, Copy)] +struct Shape { + /// Concurrent tar parts, as an object-store backup is partitioned + parts: usize, + /// Relation segments per part, one tar entry each + segments_per_part: usize, + pages_per_segment: usize, + tuples_per_page: usize, + /// int4 columns per tuple + columns: usize, + parallelism: usize, +} + +impl Default for Shape { + fn default() -> Self { + Self { + parts: 8, + segments_per_part: 4, + pages_per_segment: 256, + tuples_per_page: 100, + columns: 8, + parallelism: 4, + } + } +} + +impl Shape { + fn segments(self) -> usize { + self.parts * self.segments_per_part + } + + fn tuples(self) -> u64 { + (self.segments() * self.pages_per_segment * self.tuples_per_page) as u64 + } + + fn bytes(self) -> u64 { + (self.segments() * self.pages_per_segment * PAGE_BYTES) as u64 + } +} + +fn descriptor(filenode: u32, columns: usize) -> Arc { + RelDescriptor { + rfn: RelFileNode { + spc_node: 1663, + db_node: DB_NODE, + rel_node: filenode, + }, + oid: filenode, + toast_oid: 0, + namespace_oid: 2200, + rel_name: RelName::new("public", &format!("t{filenode}")), + kind: 'r', + persistence: 'p', + replident: ReplIdent::Default { pk_attnums: None }, + attributes: (0..columns) + .map(|i| RelAttr { + attnum: i as i16 + 1, + name: format!("c{i}"), + type_oid: INT4OID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "int4".into(), + type_byval: true, + type_len: 4, + type_align: 'i', + type_storage: 'p', + missing_text: None, + }) + .collect(), + } + .into() +} + +fn mapping(filenode: u32, columns: usize) -> TableMapping { + TableMapping { + target: TableTarget::new("bench", &format!("t{filenode}")), + columns: (0..columns) + .map(|i| ColumnMapping { + src_attnum: i as i16 + 1, + target_name: format!("c{i}"), + target_type: "Int32".into(), + }) + .collect(), + } +} + +/// One 8 KiB heap page carrying `tuples` live int4-only tuples, laid out +/// as PG writes them: line pointers up from the header, tuple bodies down +/// from the page end. +fn heap_page(tuples: usize, columns: usize, seed: i32) -> Vec { + let body = TUPLE_HEADER + 4 * columns; + let mut page = vec![0u8; PAGE_BYTES]; + let mut written = 0usize; + for i in 0..tuples { + let off = PAGE_BYTES - (i + 1) * body; + if off < SIZE_OF_PAGE_HEADER + (i + 1) * SIZE_OF_ITEM_ID { + break; + } + let xmin = 100u32 + i as u32; + page[off..off + 4].copy_from_slice(&xmin.to_le_bytes()); + page[off + 18..off + 20].copy_from_slice(&(columns as u16).to_le_bytes()); + page[off + 20..off + 22].copy_from_slice(&0u16.to_le_bytes()); + page[off + 22] = TUPLE_HEADER as u8; + for c in 0..columns { + let at = off + TUPLE_HEADER + c * 4; + let v = seed + (i as i32) * 31 + c as i32; + page[at..at + 4].copy_from_slice(&v.to_le_bytes()); + } + let slot = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; + let raw = ((off as u32) & 0x7FFF) | (LP_NORMAL << 15) | (((body as u32) & 0x7FFF) << 17); + page[slot..slot + SIZE_OF_ITEM_ID].copy_from_slice(&raw.to_le_bytes()); + written = i + 1; + } + let pd_lower = SIZE_OF_PAGE_HEADER + written * SIZE_OF_ITEM_ID; + let pd_upper = PAGE_BYTES - written * body; + page[12..14].copy_from_slice(&(pd_lower as u16).to_le_bytes()); + page[14..16].copy_from_slice(&(pd_upper as u16).to_le_bytes()); + page +} + +/// One tar part holding `segments_per_part` `base//` entries. +async fn build_part(shape: Shape, part: usize) -> Vec { + use tokio::io::AsyncWriteExt; + + let mut builder = tokio_tar::Builder::new(Vec::new()); + for s in 0..shape.segments_per_part { + let filenode = FIRST_FILENODE + (part * shape.segments_per_part + s) as u32; + let mut body = Vec::with_capacity(shape.pages_per_segment * PAGE_BYTES); + for p in 0..shape.pages_per_segment { + body.extend_from_slice(&heap_page( + shape.tuples_per_page, + shape.columns, + (p * 1_000) as i32, + )); + } + let mut header = tokio_tar::Header::new_gnu(); + header + .set_path(format!("base/{DB_NODE}/{filenode}")) + .unwrap(); + header.set_size(body.len() as u64); + header.set_mode(0o600); + header.set_entry_type(tokio_tar::EntryType::Regular); + header.set_cksum(); + builder + .append(&header, std::io::Cursor::new(body)) + .await + .unwrap(); + } + builder.finish().await.unwrap(); + let mut out = builder.into_inner().await.unwrap(); + out.flush().await.unwrap(); + out +} + +/// Object-store-shaped source over pre-built tars: one `pump_tar_to_sink` +/// per part, `parallelism` in flight, so the shared sink sees the same +/// interleaving a real fan-out gives it. +struct MemSource { + parts: Vec>, + parallelism: usize, +} + +#[async_trait::async_trait] +impl BackupSource for MemSource { + async fn run( + self: Box, + data_dir: std::path::PathBuf, + sink: Arc, + stats: Arc, + ) -> anyhow::Result<(StartInfo, EndInfo)> { + use futures::{StreamExt, TryStreamExt}; + + let start = StartInfo { + start_lsn: START_LSN, + timeline: 1, + tablespaces: Vec::::new(), + }; + let end = EndInfo { + end_lsn: START_LSN + 0x1000, + timeline: 1, + }; + sink.start(&start).await?; + let target = Arc::new(PumpTarget::new(data_dir, sink.clone(), stats)); + let MemSource { parts, parallelism } = *self; + futures::stream::iter(parts) + .map(|blob| { + let target = target.clone(); + async move { + let mut archive = tokio_tar::Archive::new(std::io::Cursor::new(blob)); + pump_tar_to_sink(&mut archive, &target).await + } + }) + .buffer_unordered(parallelism) + .try_collect::>() + .await?; + sink.finish(&end).await?; + Ok((start, end)) + } +} + +struct Report { + label: String, + elapsed_secs: f64, + tuples: u64, + bytes: u64, + detail: Vec<(&'static str, f64)>, +} + +impl Report { + fn print(&self) { + println!( + "{:<10} {:>8.3}s {:>10.1} MiB/s {:>12.0} tuples/s ({} tuples, {:.1} MiB)", + self.label, + self.elapsed_secs, + self.bytes as f64 / self.elapsed_secs / (1 << 20) as f64, + self.tuples as f64 / self.elapsed_secs, + self.tuples, + self.bytes as f64 / (1 << 20) as f64, + ); + for (name, v) in &self.detail { + println!(" {name:<26} {v:>10.3}"); + } + } +} + +async fn bench_pump(shape: Shape) -> Report { + let mut parts = Vec::with_capacity(shape.parts); + for p in 0..shape.parts { + parts.push(build_part(shape, p).await); + } + + let mut catalog = CatalogMap::new(); + for i in 0..shape.segments() { + catalog.insert(descriptor(FIRST_FILENODE + i as u32, shape.columns)); + } + + let tmp = tempfile::tempdir().unwrap(); + let cfg = BootstrapConfig::new(tmp.path().join("data")); + let progress = cfg.progress.clone(); + let source = Box::new(MemSource { + parts, + parallelism: shape.parallelism, + }); + + let started = Instant::now(); + let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog, false); + let drained = tokio::spawn(async move { + let mut n = 0u64; + while let Some(slab) = rx.recv().await { + // Touch the payload so the decode can't be optimised away + n += slab.iter().map(|t| t.columns.len() as u64).sum::(); + } + n + }); + pump.await.unwrap().unwrap(); + let touched = drained.await.unwrap(); + let elapsed = started.elapsed().as_secs_f64(); + + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64; + Report { + label: "pump".into(), + elapsed_secs: elapsed, + tuples: touched / shape.columns as u64, + bytes: ld(&progress.pump.bytes_tapped) as u64, + detail: vec![ + ("pages_walked", ld(&progress.page_walk.pages_walked)), + ("decode_secs", ld(&progress.page_walk.decode_nanos) / 1e9), + ("tap_secs", ld(&progress.pump.sink_chunk_nanos) / 1e9), + ( + "channel_block_secs", + ld(&progress.page_walk.channel_block_nanos) / 1e9, + ), + ], + } +} + +async fn bench_drain(shape: Shape) -> Report { + let rels: Vec> = (0..shape.segments()) + .map(|i| descriptor(FIRST_FILENODE + i as u32, shape.columns)) + .collect(); + let mut catalog = CatalogMap::new(); + let mut tables = std::collections::HashMap::new(); + for r in &rels { + catalog.insert(r.clone()); + tables.insert(r.rel_name.clone(), mapping(r.oid, shape.columns)); + } + let handle = walshadow::mapping::mapping_handle(tables.into_iter().collect()); + + let (msg_tx, ack, tail) = + walshadow::pipeline::tail::spawn_null(Arc::new(Monotone::::new(0))); + let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::>(16); + + let per_rel = shape.pages_per_segment as u64 * shape.tuples_per_page as u64; + let columns = shape.columns; + let feeder = tokio::spawn(async move { + for r in rels { + // Slab at the walk's grain so the drain sees production shape + for slab_start in (0..per_rel).step_by(shape.tuples_per_page * 16) { + let end = (slab_start + (shape.tuples_per_page * 16) as u64).min(per_rel); + let slab: Vec = (slab_start..end) + .map(|i| BackfillTuple { + rfn: r.rfn, + xid: 100 + i as u32, + xmax: 0, + infomask: 0, + source_lsn: START_LSN, + blkno: (i / 100) as u32, + offnum: (i % 100) as u16 + 1, + columns: (0..columns) + .map(|c| Some(ColumnValue::Int4(i as i32 + c as i32))) + .collect(), + }) + .collect(); + if tup_tx.send(slab).await.is_err() { + return; + } + } + } + }); + + let tmp = tempfile::tempdir().unwrap(); + let stats = Arc::new(walshadow::ch_emitter::EmitterStats::default()); + let started = Instant::now(); + let outcome = bootstrap::drain( + tup_rx, + catalog, + handle, + msg_tx.clone(), + ack.clone(), + stats.clone(), + ToastResolver::disabled(), + walshadow::spool::DeferredSpool::new( + tmp.path().join("deferred.bin"), + walshadow::spool::DEFERRED_SPOOL_MEM_MAX, + ), + Default::default(), + None, + std::collections::HashSet::new(), + ) + .await + .unwrap(); + let elapsed = started.elapsed().as_secs_f64(); + feeder.await.unwrap(); + drop(msg_tx); + drop(ack); + tail.join().await; + + Report { + label: "drain".into(), + elapsed_secs: elapsed, + tuples: outcome.rows_routed, + // Int4 payload only; the interesting rate here is rows, not bytes + bytes: outcome.rows_routed * 4 * shape.columns as u64, + detail: vec![("seqs", outcome.next_seq as f64)], + } +} + +fn usage() -> ! { + eprintln!( + "usage: bootstrap_pump [--list] [--stage pump|drain|all] [--parts N] \ + [--segments-per-part N] [--pages N] [--tuples-per-page N] [--columns N] \ + [--parallelism N]" + ); + std::process::exit(2) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + // nextest --all-targets lists benches by running them with --list + if args.iter().any(|a| a == "--list") { + println!("bootstrap_pump: benchmark"); + return; + } + let mut shape = Shape::default(); + let mut stage = "all".to_string(); + let mut i = 0; + while i < args.len() { + let val = |s: &Option<&String>| -> usize { + s.and_then(|v| v.parse().ok()).unwrap_or_else(|| usage()) + }; + let next = args.get(i + 1); + match args[i].as_str() { + "--stage" => stage = next.cloned().unwrap_or_else(|| usage()), + "--parts" => shape.parts = val(&next).max(1), + "--segments-per-part" => shape.segments_per_part = val(&next).max(1), + "--pages" => shape.pages_per_segment = val(&next).max(1), + "--tuples-per-page" => shape.tuples_per_page = val(&next).max(1), + "--columns" => shape.columns = val(&next).max(1), + "--parallelism" => shape.parallelism = val(&next).max(1), + // cargo bench passes --bench through + "--bench" => { + i += 1; + continue; + } + _ => usage(), + } + i += 2; + } + + let rt = tokio::runtime::Runtime::new().unwrap(); + println!( + "shape: {} segments x {} pages x {} tuples x {} int4 cols, {} parts, parallelism {}", + shape.segments(), + shape.pages_per_segment, + shape.tuples_per_page, + shape.columns, + shape.parts, + shape.parallelism, + ); + println!( + " {:.1} MiB of heap, {} tuples", + shape.bytes() as f64 / (1 << 20) as f64, + shape.tuples(), + ); + if stage == "pump" || stage == "all" { + rt.block_on(bench_pump(shape)).print(); + } + if stage == "drain" || stage == "all" { + rt.block_on(bench_drain(shape)).print(); + } +} diff --git a/benches/oracle_batch.rs b/benches/oracle_batch.rs new file mode 100644 index 00000000..03e49188 --- /dev/null +++ b/benches/oracle_batch.rs @@ -0,0 +1,392 @@ +//! Oracle batch cost, by stage. +//! +//! - `frame`: the two payload-sized memory passes a request used to make, no +//! Postgres. One is the request copy into a second buffer the bridge +//! framed; the other is the zeroing pass a fresh response buffer pays +//! before `read_exact` overwrites it. Both scale with the batch, and +//! `ORACLE_BATCH_SEAL_BYTES` puts that at 32 MiB +//! - `roundtrip`: a real shadow Postgres and a real `ENCODE_NATIVE`. Reports +//! what one batch costs end to end, alone and with the bridge pool +//! saturated, so the memory passes above have something to be a fraction +//! of. Also prices a column of `Literal` cells — bytes the daemon already +//! rendered — against building the same ClickHouse column locally +//! +//! ```text +//! cargo bench --bench oracle_batch -- --stage frame --bytes 33554432 +//! cargo bench --bench oracle_batch -- --stage roundtrip --rows 100000 --workers 4 +//! ``` +//! +//! `roundtrip` needs `initdb` on PATH and `pgext/walshadow.so` built against +//! that major; it says so and skips otherwise. +//! +//! `harness = false`, so nextest lists it by answering `--list`. + +// Matches `walshadow-stream`, whose allocator is not glibc's. Request +// buffers here are allocated and freed at 32 MiB, which is exactly where +// allocators differ most +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +use std::path::PathBuf; +use std::process::Command; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clickhouse_c::{Allocator, ColumnBuilder}; +use walshadow::bridge::Bridge; +use walshadow::oracle::{Oracle, OracleCell, OracleColumnBuf, OracleRequestColumn}; +use walshadow::schema::NUMERICOID; +use walshadow::shadow::{BridgeConf, Shadow, ShadowConfig}; + +/// Port outside the integration suites' allocation +const PG_PORT: u16 = 55450; +/// Short-form numeric for 42: header 0x8000, one base-10000 digit +fn numeric_42() -> Vec { + let mut out = 0x8000u16.to_le_bytes().to_vec(); + out.extend_from_slice(&42i16.to_le_bytes()); + out +} + +/// WKT a PostGIS 2-D point renders to, ie a `Literal` cell +const WKT: &[u8] = b"POINT(-73.987654 40.748817)"; + +struct Args { + stage: String, + bytes: usize, + rows: usize, + workers: usize, + iters: usize, +} + +impl Default for Args { + fn default() -> Self { + Self { + stage: "all".into(), + bytes: 32 << 20, + rows: 100_000, + workers: 4, + iters: 8, + } + } +} + +fn usage() -> ! { + eprintln!( + "usage: --stage frame|roundtrip|all [--bytes N] [--rows N] [--workers N] [--iters N]" + ); + std::process::exit(2); +} + +/// Mean over `iters`, minus a warmup pass whose cost is first-touch +fn timed(iters: usize, mut f: impl FnMut()) -> Duration { + f(); + let started = Instant::now(); + for _ in 0..iters { + f(); + } + started.elapsed() / iters as u32 +} + +fn mib_per_s(bytes: usize, d: Duration) -> f64 { + bytes as f64 / (1 << 20) as f64 / d.as_secs_f64() +} + +/// The two payload-sized passes a batch no longer makes +fn bench_frame(args: &Args) { + let n = args.bytes; + let src = vec![0xa5u8; n]; + + // Request side, before: payload into its own buffer, then a second + // buffer of 4 + len the whole payload is copied into so one write_all + // ships it. After: cells go straight past a reserved prefix, so neither + // the allocation nor the copy happens + let copy = timed(args.iters, || { + let mut frame = Vec::with_capacity(5 + n); + frame.extend_from_slice(&[0u8; 5]); + frame.extend_from_slice(&src); + std::hint::black_box(frame.len()); + }); + + // Response side: `vec![0u8; len]` zeroes bytes read_exact then + // overwrites. A recycled buffer read into its spare capacity pays + // neither the allocation nor the zeroing + let fresh = timed(args.iters, || { + let mut v = vec![0u8; n]; + v.copy_from_slice(&src); + std::hint::black_box(v.len()); + }); + let mut scratch: Vec = Vec::with_capacity(n); + let recycled = timed(args.iters, || { + scratch.clear(); + scratch.extend_from_slice(&src); + std::hint::black_box(scratch.len()); + }); + + println!( + "frame stage, {:.0} MiB payload", + n as f64 / (1 << 20) as f64 + ); + println!( + " request second buffer + copy {:>8.3} ms ({:.0} MiB/s)", + copy.as_secs_f64() * 1e3, + mib_per_s(n, copy), + ); + println!( + " response vec![0u8; len] + fill {:>8.3} ms ({:.0} MiB/s)", + fresh.as_secs_f64() * 1e3, + mib_per_s(n, fresh), + ); + println!( + " response recycled + fill {:>8.3} ms ({:.0} MiB/s)", + recycled.as_secs_f64() * 1e3, + mib_per_s(n, recycled), + ); + println!( + " zeroing pass alone {:>8.3} ms", + (fresh.as_secs_f64() - recycled.as_secs_f64()) * 1e3, + ); +} + +fn pg_available() -> bool { + Command::new("initdb") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn pgext_dir() -> Option { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("pgext"); + dir.join("walshadow.so").is_file().then_some(dir) +} + +struct StopOnDrop { + sh: Shadow, +} + +impl Drop for StopOnDrop { + fn drop(&mut self) { + let _ = self.sh.stop(); + } +} + +fn start_shadow(tmp: &tempfile::TempDir, lib_dir: PathBuf, workers: usize) -> StopOnDrop { + let mut cfg = ShadowConfig::new(tmp.path().join("data"), tmp.path().join("filtered")); + cfg.port = PG_PORT; + cfg.socket_dir = tmp.path().join("sock"); + cfg.ctl_timeout = Duration::from_secs(60); + let mut bridge = BridgeConf::in_dir(&cfg.socket_dir); + bridge.library_dir = Some(lib_dir); + bridge.workers = workers; + cfg.bridge = Some(bridge); + std::fs::create_dir_all(&cfg.filter_out_dir).unwrap(); + std::fs::create_dir_all(&cfg.socket_dir).unwrap(); + let sh = Shadow::new(cfg); + sh.initdb().expect("initdb"); + sh.write_base_conf().expect("write_base_conf"); + sh.start().expect("start"); + StopOnDrop { sh } +} + +fn cells(oid: u32, body: &[u8], rows: usize, literal: bool) -> OracleColumnBuf { + let mut buf = OracleColumnBuf::new(oid, -1); + for _ in 0..rows { + buf.push(if literal { + OracleCell::Literal(body.to_vec()) + } else { + OracleCell::DiskRaw(body.to_vec()) + }); + } + buf +} + +async fn one_batch(oracle: &Oracle, buf: &OracleColumnBuf, rows: usize) -> Duration { + let columns = [OracleRequestColumn { + ordinal: 0, + name: "c0", + target_type: "String", + buf, + }]; + let started = Instant::now(); + let block = oracle + .encode_batch(&columns, rows, Allocator::stdlib()) + .await + .expect("oracle answers"); + let elapsed = started.elapsed(); + assert_eq!( + block + .column(0) + .and_then(|c| c.string()) + .expect("string") + .0 + .len(), + rows, + ); + elapsed +} + +/// Mean batch latency, `iters` batches in flight `concurrency` at a time +async fn batches( + oracle: Arc, + buf: Arc, + rows: usize, + iters: usize, + concurrency: usize, +) -> (Duration, Duration) { + let started = Instant::now(); + let mut sum = Duration::ZERO; + let mut left = iters; + while left > 0 { + let wave = concurrency.min(left); + let mut set = Vec::new(); + for _ in 0..wave { + let oracle = oracle.clone(); + let buf = buf.clone(); + set.push(tokio::spawn( + async move { one_batch(&oracle, &buf, rows).await }, + )); + } + for h in set { + sum += h.await.unwrap(); + } + left -= wave; + } + (sum / iters as u32, started.elapsed() / iters as u32) +} + +/// `ColumnBuilder::string` over the same cells, ie what a pure-`Literal` +/// column would cost if it never left the daemon +fn local_string_column(rows: usize, body: &[u8]) -> Duration { + let iters = 8; + timed(iters, || { + let mut offsets: Vec = Vec::with_capacity(rows); + let mut data: Vec = Vec::with_capacity(rows * body.len()); + for _ in 0..rows { + data.extend_from_slice(body); + offsets.push(data.len() as u64); + } + let col = ColumnBuilder::string(&offsets, &data, rows).expect("string column"); + std::hint::black_box(col.n_rows()); + }) +} + +async fn bench_roundtrip(args: &Args) { + let Some(lib_dir) = pgext_dir() else { + println!("roundtrip stage: skip, pgext/walshadow.so missing (make -C pgext)"); + return; + }; + if !pg_available() { + println!("roundtrip stage: skip, no initdb on PATH"); + return; + } + let tmp = tempfile::tempdir().unwrap(); + let guard = start_shadow(&tmp, lib_dir, args.workers); + let socket = guard.sh.bridge_socket().expect("bridge configured"); + + let disk = Arc::new(cells(NUMERICOID, &numeric_42(), args.rows, false)); + let literal = Arc::new(cells(NUMERICOID, WKT, args.rows, true)); + let req_bytes = |b: &OracleColumnBuf| b.approx_size(); + + for workers in [1, args.workers] { + let bridge = Arc::new( + walshadow::bridge::connect_with_budget(socket, workers, Duration::from_secs(30)) + .await + .expect("bridge connect"), + ); + let oracle = Arc::new(Oracle::new(bridge.clone())); + let serial = batches(oracle.clone(), disk.clone(), args.rows, args.iters, 1).await; + let parallel = batches( + oracle.clone(), + disk.clone(), + args.rows, + args.iters, + workers.max(1), + ) + .await; + println!( + "roundtrip stage, {} DiskRaw numeric cells ({:.1} MiB request), {} sockets", + args.rows, + req_bytes(&disk) as f64 / (1 << 20) as f64, + bridge.pool_size(), + ); + println!( + " one batch at a time {:>8.2} ms/batch", + serial.1.as_secs_f64() * 1e3, + ); + println!( + " {:>2} in flight {:>8.2} ms/batch wall, {:.2} ms latency", + workers, + parallel.1.as_secs_f64() * 1e3, + parallel.0.as_secs_f64() * 1e3, + ); + } + + // Cells the daemon already rendered, which the resolver now builds + // locally: this is what shipping them used to cost + let bridge = Arc::new( + walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) + .await + .expect("bridge connect"), + ); + let oracle = Arc::new(Oracle::new(bridge)); + let shipped = batches(oracle, literal.clone(), args.rows, args.iters, 1).await; + let built = local_string_column(args.rows, WKT); + println!( + "literal column, {} cells ({:.1} MiB)", + args.rows, + req_bytes(&literal) as f64 / (1 << 20) as f64, + ); + println!( + " through the oracle {:>8.2} ms/batch", + shipped.1.as_secs_f64() * 1e3, + ); + println!( + " built locally {:>8.2} ms/batch", + built.as_secs_f64() * 1e3, + ); +} + +fn main() { + let argv: Vec = std::env::args().skip(1).collect(); + // nextest --all-targets lists benches by running them with --list + if argv.iter().any(|a| a == "--list") { + println!("oracle_batch: benchmark"); + return; + } + let mut args = Args::default(); + let mut i = 0; + while i < argv.len() { + let val = |s: &Option<&String>| -> usize { + s.and_then(|v| v.parse().ok()).unwrap_or_else(|| usage()) + }; + let next = argv.get(i + 1); + match argv[i].as_str() { + "--stage" => args.stage = next.cloned().unwrap_or_else(|| usage()), + "--bytes" => args.bytes = val(&next).max(1), + "--rows" => args.rows = val(&next).max(1), + "--workers" => args.workers = val(&next).clamp(1, 8), + "--iters" => args.iters = val(&next).max(1), + // cargo bench passes --bench through + "--bench" => { + i += 1; + continue; + } + _ => usage(), + } + i += 2; + } + + if args.stage == "frame" || args.stage == "all" { + bench_frame(&args); + } + if args.stage == "roundtrip" || args.stage == "all" { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(bench_roundtrip(&args)); + } +} + +/// Silences the unused-import lint when only `frame` is compiled in +#[allow(dead_code)] +fn _bridge_type_is_used(_: &Bridge) {} diff --git a/pgext/worker.c b/pgext/worker.c index 4f9fd40e..a52be27a 100644 --- a/pgext/worker.c +++ b/pgext/worker.c @@ -43,9 +43,22 @@ PG_MODULE_MAGIC; #define WS_MAX_CONNS 8 +/* + * Each worker serves one request at a time, so oracle throughput is one + * backend's conversion rate. `walshadow.bridge_workers` registers copies; + * worker 0 keeps the bare `socket_path` so a single-worker deployment and + * every catalog read are untouched, worker i listens on `socket_path.i`. + */ +#define WS_MAX_WORKERS 8 #define WS_LISTEN_BACKLOG 16 #define WS_IDLE_POLL_MS 1000 #define WS_MAX_SCAN_OIDS 65536 +/* + * A 32 MiB request against default socket buffers costs hundreds of + * EAGAIN round trips; ask for a wide window and accept whatever the + * kernel grants (it halves the request and clamps to wmem_max). + */ +#define WS_SOCKBUF_BYTES (4 * 1024 * 1024) /* wait-set positions, in the order ws_build_wait_set adds them */ #define WS_POS_LATCH 0 @@ -53,10 +66,27 @@ PG_MODULE_MAGIC; #define WS_POS_LISTEN 2 #define WS_POS_CONN0 3 +/* ... and in the per-connection io set */ +#define WS_IO_POS_LATCH 0 +#define WS_IO_POS_PM_DEATH 1 +#define WS_IO_POS_SOCKET 2 + +/* + * One accepted connection. `io` is created once and its socket event mask + * flipped between readable and writeable, because building a fresh epoll + * set per EAGAIN is what a large transfer would otherwise pay for. + */ +typedef struct WsConn +{ + pgsocket fd; + WaitEventSet *io; +} WsConn; + PGDLLEXPORT void ws_worker_main(Datum main_arg); static char *ws_socket_path = NULL; static char *ws_database = NULL; +static int ws_bridge_workers = 1; static int ws_io_timeout_ms = 30000; static int ws_lock_timeout_ms = 1000; @@ -175,20 +205,34 @@ ws_listen(const char *path) return fd; } +static WaitEventSet * +ws_create_wait_set(int nevents) +{ +#if PG_VERSION_NUM >= 170000 + return CreateWaitEventSet(NULL, nevents); +#else + return CreateWaitEventSet(TopMemoryContext, nevents); +#endif +} + /* - * Wait for `event` on one socket. `false` means the caller should abandon the - * connection: shutdown was requested. + * Wait for `event` on one connection, reusing its own wait set. `false` + * means the caller should abandon the connection: shutdown was requested. */ static bool -ws_wait_socket(pgsocket fd, int event, long timeout_ms) +ws_wait_conn(WsConn *conn, int event, long timeout_ms) { - int rc; + WaitEvent events[3]; + int nready; + int i; - rc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT | event, - fd, timeout_ms, PG_WAIT_EXTENSION); - if (rc & WL_LATCH_SET) + ModifyWaitEvent(conn->io, WS_IO_POS_SOCKET, event, NULL); + nready = WaitEventSetWait(conn->io, timeout_ms, events, + lengthof(events), PG_WAIT_EXTENSION); + for (i = 0; i < nready; i++) { + if (events[i].pos != WS_IO_POS_LATCH) + continue; ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); if (ConfigReloadPending) @@ -203,10 +247,11 @@ ws_wait_socket(pgsocket fd, int event, long timeout_ms) } static bool -ws_read_exact(pgsocket fd, char *buf, size_t len) +ws_read_exact(WsConn *conn, char *buf, size_t len) { size_t got = 0; TimestampTz deadline; + pgsocket fd = conn->fd; deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), ws_io_timeout_ms); @@ -242,17 +287,18 @@ ws_read_exact(pgsocket fd, char *buf, size_t len) ws_io_timeout_ms))); return false; } - if (!ws_wait_socket(fd, WL_SOCKET_READABLE, wait_ms)) + if (!ws_wait_conn(conn, WL_SOCKET_READABLE, wait_ms)) return false; } return true; } static bool -ws_write_all(pgsocket fd, const char *buf, size_t len) +ws_write_all(WsConn *conn, const char *buf, size_t len) { size_t sent = 0; TimestampTz deadline; + pgsocket fd = conn->fd; deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), ws_io_timeout_ms); @@ -288,7 +334,7 @@ ws_write_all(pgsocket fd, const char *buf, size_t len) ws_io_timeout_ms))); return false; } - if (!ws_wait_socket(fd, WL_SOCKET_WRITEABLE, wait_ms)) + if (!ws_wait_conn(conn, WL_SOCKET_WRITEABLE, wait_ms)) return false; } return true; @@ -448,7 +494,7 @@ ws_dispatch(StringInfo req, StringInfo resp) * `false` closes the connection. */ static bool -ws_serve_request(pgsocket fd) +ws_serve_request(WsConn *conn) { uint32 hdr; uint32 len; @@ -457,7 +503,7 @@ ws_serve_request(pgsocket fd) MemoryContext oldctx; bool ok = false; - if (!ws_read_exact(fd, (char *) &hdr, sizeof(hdr))) + if (!ws_read_exact(conn, (char *) &hdr, sizeof(hdr))) return false; len = pg_ntoh32(hdr); @@ -476,7 +522,7 @@ ws_serve_request(pgsocket fd) initStringInfo(&req); enlargeStringInfo(&req, (int) len); - if (ws_read_exact(fd, req.data, len)) + if (ws_read_exact(conn, req.data, len)) { req.len = (int) len; req.data[len] = '\0'; @@ -487,8 +533,8 @@ ws_serve_request(pgsocket fd) ws_dispatch(&req, &resp); hdr = pg_hton32((uint32) resp.len); - ok = ws_write_all(fd, (char *) &hdr, sizeof(hdr)) && - ws_write_all(fd, resp.data, (size_t) resp.len); + ok = ws_write_all(conn, (char *) &hdr, sizeof(hdr)) && + ws_write_all(conn, resp.data, (size_t) resp.len); } MemoryContextSwitchTo(oldctx); @@ -501,32 +547,54 @@ ws_serve_request(pgsocket fd) * ------------------------------------------------------------------------- */ /* - * Rebuilt per iteration: the wait-set API has no portable event removal, and - * this loop is idle-dominated. + * The wait-set API has no portable event removal, so the set is rebuilt on + * a membership change and reused across every iteration in between. Under + * load that is one epoll set per connect/disconnect rather than per request. */ static WaitEventSet * -ws_build_wait_set(pgsocket listen_fd, const pgsocket *conns, int nconns) +ws_build_wait_set(pgsocket listen_fd, const WsConn *conns, int nconns) { - WaitEventSet *set; + WaitEventSet *set = ws_create_wait_set(nconns + WS_POS_CONN0); int i; -#if PG_VERSION_NUM >= 170000 - set = CreateWaitEventSet(NULL, nconns + WS_POS_CONN0); -#else - set = CreateWaitEventSet(CurrentMemoryContext, nconns + WS_POS_CONN0); -#endif AddWaitEventToSet(set, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); AddWaitEventToSet(set, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); AddWaitEventToSet(set, WL_SOCKET_READABLE, listen_fd, NULL, NULL); for (i = 0; i < nconns; i++) - AddWaitEventToSet(set, WL_SOCKET_READABLE, conns[i], NULL, NULL); + AddWaitEventToSet(set, WL_SOCKET_READABLE, conns[i].fd, NULL, NULL); return set; } +/* + * Widen the socket buffers so a multi-megabyte frame crosses in a handful + * of syscalls. Advisory: a kernel that refuses leaves the default, which + * only costs more EAGAIN waits. + */ static void -ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) +ws_widen_sockbufs(pgsocket fd) +{ + int want = WS_SOCKBUF_BYTES; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (char *) &want, sizeof(want)) < 0 || + setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (char *) &want, sizeof(want)) < 0) + ereport(DEBUG1, + (errcode_for_socket_access(), + errmsg("walshadow: could not widen socket buffers: %m"))); +} + +/* + * Membership generation, bumped on every accept and drop. The serve loop + * keys its cached wait set off this, not off nconns: an accept and a drop + * in the same iteration leave nconns unchanged over a different fd set, + * and a stale set would then poll a closed fd and miss the new one. + */ +static uint64 ws_conn_gen = 0; + +static bool +ws_accept(pgsocket listen_fd, WsConn *conns, int *nconns) { pgsocket fd; + WsConn *conn; fd = accept(listen_fd, NULL, NULL); if (fd == PGINVALID_SOCKET) @@ -535,7 +603,7 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) ereport(LOG, (errcode_for_socket_access(), errmsg("walshadow: accept failed: %m"))); - return; + return false; } if (*nconns >= WS_MAX_CONNS) { @@ -543,7 +611,7 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) (errmsg("walshadow: refusing connection, %d already open", WS_MAX_CONNS))); closesocket(fd); - return; + return false; } if (!pg_set_noblock(fd)) { @@ -551,27 +619,39 @@ ws_accept(pgsocket listen_fd, pgsocket *conns, int *nconns) (errcode_for_socket_access(), errmsg("walshadow: could not set client socket non-blocking: %m"))); closesocket(fd); - return; + return false; } - conns[(*nconns)++] = fd; + ws_widen_sockbufs(fd); + + conn = &conns[(*nconns)++]; + ws_conn_gen++; + conn->fd = fd; + conn->io = ws_create_wait_set(3); + AddWaitEventToSet(conn->io, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); + AddWaitEventToSet(conn->io, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, NULL, NULL); + AddWaitEventToSet(conn->io, WL_SOCKET_READABLE, fd, NULL, NULL); + return true; } static void -ws_drop_conn(pgsocket *conns, int *nconns, int idx) +ws_drop_conn(WsConn *conns, int *nconns, int idx) { - closesocket(conns[idx]); + FreeWaitEventSet(conns[idx].io); + closesocket(conns[idx].fd); conns[idx] = conns[--*nconns]; + ws_conn_gen++; } static void ws_serve_loop(pgsocket listen_fd) { - pgsocket conns[WS_MAX_CONNS]; + WsConn conns[WS_MAX_CONNS]; int nconns = 0; + WaitEventSet *set = NULL; + uint64 set_gen = 0; for (;;) { - WaitEventSet *set; WaitEvent events[WS_MAX_CONNS + WS_POS_CONN0]; int nready; int i; @@ -586,10 +666,15 @@ ws_serve_loop(pgsocket listen_fd) if (ShutdownRequestPending) break; - set = ws_build_wait_set(listen_fd, conns, nconns); + if (set == NULL || set_gen != ws_conn_gen) + { + if (set != NULL) + FreeWaitEventSet(set); + set = ws_build_wait_set(listen_fd, conns, nconns); + set_gen = ws_conn_gen; + } nready = WaitEventSetWait(set, WS_IDLE_POLL_MS, events, lengthof(events), PG_WAIT_EXTENSION); - FreeWaitEventSet(set); for (i = 0; i < nready; i++) { @@ -611,12 +696,14 @@ ws_serve_loop(pgsocket listen_fd) if (serve >= 0 && serve < nconns) { pgstat_report_activity(STATE_RUNNING, "walshadow request"); - if (!ws_serve_request(conns[serve])) + if (!ws_serve_request(&conns[serve])) ws_drop_conn(conns, &nconns, serve); pgstat_report_activity(STATE_IDLE, NULL); } } + if (set != NULL) + FreeWaitEventSet(set); while (nconns > 0) ws_drop_conn(conns, &nconns, 0); closesocket(listen_fd); @@ -625,8 +712,10 @@ ws_serve_loop(pgsocket listen_fd) void ws_worker_main(Datum main_arg) { + int idx = DatumGetInt32(main_arg); pgsocket listen_fd; char buf[32]; + char path[MAXPGPATH]; pqsignal(SIGTERM, SignalHandlerForShutdownRequest); pqsignal(SIGHUP, SignalHandlerForConfigReload); @@ -659,10 +748,14 @@ ws_worker_main(Datum main_arg) "walshadow request", ALLOCSET_DEFAULT_SIZES); - listen_fd = ws_listen(ws_socket_path); + if (idx == 0) + strlcpy(path, ws_socket_path, sizeof(path)); + else + snprintf(path, sizeof(path), "%s.%d", ws_socket_path, idx); + listen_fd = ws_listen(path); ereport(LOG, (errmsg("walshadow bridge listening on \"%s\" (proto %d)", - ws_socket_path, WS_PROTO_VERSION))); + path, WS_PROTO_VERSION))); ws_serve_loop(listen_fd); @@ -680,6 +773,7 @@ void _PG_init(void) { BackgroundWorker worker; + int i; /* * Worker registration and its postmaster-scoped GUCs are only legal @@ -716,20 +810,33 @@ _PG_init(void) 1000, 0, INT_MAX, PGC_POSTMASTER, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable("walshadow.bridge_workers", + "Bridge workers to register.", + "Worker 0 listens on socket_path, worker i on " + "socket_path.i. Each serves one request at a " + "time, so this bounds concurrent decode.", + &ws_bridge_workers, + 1, 1, WS_MAX_WORKERS, + PGC_POSTMASTER, 0, + NULL, NULL, NULL); MarkGUCPrefixReserved("walshadow"); if (ws_socket_path == NULL || ws_socket_path[0] == '\0') return; - memset(&worker, 0, sizeof(worker)); - worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - /* Catalog reads need a database connection, so not before consistency */ - worker.bgw_start_time = BgWorkerStart_ConsistentState; - worker.bgw_restart_time = 5; - strlcpy(worker.bgw_library_name, "walshadow", BGW_MAXLEN); - strlcpy(worker.bgw_function_name, "ws_worker_main", BGW_MAXLEN); - strlcpy(worker.bgw_name, "walshadow bridge", BGW_MAXLEN); - strlcpy(worker.bgw_type, "walshadow bridge", BGW_MAXLEN); - RegisterBackgroundWorker(&worker); + for (i = 0; i < ws_bridge_workers; i++) + { + memset(&worker, 0, sizeof(worker)); + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; + /* Catalog reads need a database connection, so not before consistency */ + worker.bgw_start_time = BgWorkerStart_ConsistentState; + worker.bgw_restart_time = 5; + worker.bgw_main_arg = Int32GetDatum(i); + strlcpy(worker.bgw_library_name, "walshadow", BGW_MAXLEN); + strlcpy(worker.bgw_function_name, "ws_worker_main", BGW_MAXLEN); + snprintf(worker.bgw_name, BGW_MAXLEN, "walshadow bridge %d", i); + strlcpy(worker.bgw_type, "walshadow bridge", BGW_MAXLEN); + RegisterBackgroundWorker(&worker); + } } diff --git a/plans/GLOSSARY.md b/plans/GLOSSARY.md index 478b4cb2..9f70d23f 100644 --- a/plans/GLOSSARY.md +++ b/plans/GLOSSARY.md @@ -208,9 +208,10 @@ so does the whole segment holding it, since a fork copies the ancestor prefix into a descendant-named file ([failover.md](failover.md)) **FileAction (Keep / Skip / Tap)** — per-backup-file sink decision: land -body under data dir, drain unread, or stream body through `chunk()` -with nothing landing (page walk, pg_xact accumulation) -([bootstrap.md](bootstrap.md)) +body under data dir, drain unread, or stream body into an owned +`EntrySink` with nothing landing (page walk, pg_xact accumulation). +Owned rather than a shared-state lookup so concurrent tar parts share no +lock on the body path ([bootstrap.md](bootstrap.md)) **filter** — per-record keep/drop engine on WalStream: kept records pass (catalog-only blocks retained, CRC32C recomputed), drops NOOP-rewrite in @@ -349,8 +350,8 @@ whole overlay subsystem; `config_table.replicate` opts one table into replication, triggering backfill per `initial_load` ([table selection guide](../docs/table-selection.md), [add_table.md](add_table.md)) -**oracle** — resolver for types walshadow does not decode: shadow PG -converts on-disk bytes into a ClickHouse Native column through the bridge +**oracle** — converter for types walshadow does not decode: shadow PG +turns on-disk bytes into a ClickHouse Native column through the bridge worker's `ENCODE_NATIVE` op, one request per sealed batch. No fallback, a cell it cannot convert fails the batch ([oracle.md](oracle.md)) @@ -400,8 +401,8 @@ backup files, patched with commit/abort records harvested from gap-WAL pre-scan; backs the visibility gate ([add_table.md](add_table.md)) **pipeline** — parallel decode+insert tail: reorder → decode ×M → -batcher → inserter ×N → ack watermark, in `src/pipeline/`; stands up -only with `--ch-config` ([emitter.md](emitter.md)) +batcher → resolve ×W → inserter ×N → ack watermark, in `src/pipeline/`; +stands up only with `--ch-config` ([emitter.md](emitter.md)) **placed** — decoder's per-seq report that all a xact's rows are routed to batcher; seq is done at `placed == acked` @@ -461,6 +462,12 @@ keyless `Default` ([shadow.md](shadow.md)) `RM_XACT_ID` records; assigns seqs, dispatches decode jobs, owns barriers ([emitter.md](emitter.md)) +**resolver pool** — ×W tasks between batcher and inserters answering +sealed batches' oracle columns, W = the bridge's pool width; overlaps a +bridge round trip with other inserters' queries, and builds columns of +already-rendered cells locally instead of shipping them +([emitter.md](emitter.md), [oracle.md](oracle.md)) + **restore_command fallback** — shadow's archive channel (`cp /%f %p`) at segment cadence when walsender wire drops or a slow connection is cut off ([shadow.md](shadow.md)) @@ -570,9 +577,9 @@ or `match` pattern) overrides per relation ([emitter.md](emitter.md), [destination tables guide](../docs/destination-tables.md)) -**tail** — reusable batcher + inserter pool + ack collector unit; WAL -pipeline and bootstrap drain feed the identical tail, `tail.finish` -seals partials and waits all seqs durable +**tail** — reusable batcher + resolver pool + inserter pool + ack +collector unit; WAL pipeline and bootstrap drain feed the identical tail, +`tail.finish` seals partials and waits all seqs durable ([emitter.md](emitter.md), [bootstrap.md](bootstrap.md)) **Tier 1 / 2 / 3** — decoder type matrix: fixed-width (`type_len > 0`) diff --git a/plans/bootstrap.md b/plans/bootstrap.md index 594d4b4a..18e1a51b 100644 --- a/plans/bootstrap.md +++ b/plans/bootstrap.md @@ -43,9 +43,10 @@ for rendered diagram. Five clusters top→bottom: - user heap → `PageWalkSink` (Tap) → decoded 8 KiB at a time - denylist contents → Skip; denylist dir entries themselves → Keep as empty dirs -3. **Drain → CH** (concurrent with step 2) — `PageWalkSink` ships - `BackfillTuple`s through bounded mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP - = 256`, backpressures the tar pump) to +3. **Drain → CH** (concurrent with step 2) — each `PageWalkEntry` ships + slabs of `BackfillTuple`s through a bounded mpsc + (`BOOTSTRAP_TUPLE_CHANNEL_CAP` slabs of `SLAB_BYTES`, backpressures + the tar pump) to `pipeline::bootstrap::drain`, which synthesizes rows `{ op=Insert, commit_lsn=start_lsn }` against the snapshot `CatalogMap` and routes into the shared insert tail (batcher + @@ -160,11 +161,16 @@ pub trait BackupSource: Send { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)>; } ``` +`stats` is the pump's stage attribution (`bytes_tapped`, +`sink_chunk_nanos`), shared with the orchestrator so `/metrics` reads it +while the pump runs. + Public types: - `StartInfo { start_lsn, timeline, tablespaces }` — mirrors @@ -179,9 +185,13 @@ Public types: - `FileMeta { path, size, mode, kind }` — `path` cluster-relative, sanitized against `..` / absolute-root at source-impl boundary (`tar_entry_meta` returns `Ok(None)` on parent-dir traversal) -- `FileAction::{Keep, Skip, Tap}` — sink decision per `begin()`. Keep: - source writes body under `data_dir`; Skip: drain body unread; Tap: - stream body bytes through `chunk()` callbacks, nothing lands +- `FileAction::{Keep, Skip, Tap(Box)}` — sink decision per + `begin()`. Keep: source writes body under `data_dir`; Skip: drain body + unread; Tap: source streams body bytes into the returned owned entry + sink, nothing lands +- `EntrySink` — one tap entry's body consumer, `chunk` then `end`. Owned + by the source task driving that entry, so concurrent entries share no + lock on the body path Per-source guarantees in `src/backup_source.rs` module docs: @@ -194,14 +204,20 @@ Per-source guarantees in `src/backup_source.rs` module docs: 4. `finish()` fires after the last `end()`, carries `end_lsn` 5. Paths are cluster-relative & traversal-safe -Sink trait surface (`BackupSink`): `#[async_trait]` `start` / `begin` -/ `chunk` / `end` / `finish`, `Send` so ObjectStore worker pool can -share `Arc>`. Async surface is load-bearing: -`chunk` fires inside tokio runtime context source drives, and -PageWalkSink's bounded `mpsc::Sender::send(...).await` there is what -backpressures the tar pump against drain throughput (async surface + -bounded channel exist precisely to get this bound; a sync trait + -unbounded channel would not) +Sink trait surface (`BackupSink`): `#[async_trait]` `start` / `begin` / +`finish`, all `&self`, `Send + Sync` so the ObjectStore worker pool +shares one `Arc` with no lock. Routing reads immutable +state and bumps atomics; the body path lives on the per-entry +`EntrySink` the `Tap` decision hands back. A sink that instead held a +shared lock across a body read would serialize every other entry behind +it, which is why the decision is owned rather than a lookup. + +Async surface is load-bearing: `EntrySink::chunk` fires inside the tokio +runtime context the source drives, and `PageWalkEntry`'s bounded +`mpsc::Sender::send(...).await` there is what backpressures the tar pump +against drain throughput (async surface + bounded channel exist +precisely to get this bound; a sync trait + unbounded channel would +not) ## Two source impls @@ -245,11 +261,20 @@ primitives against `DynStorage` bucket (wal-g-compatible layout): (default `min(4, num_cpus)`, overridden by `--bootstrap-object-store-parallelism` / `[bootstrap] object_store_parallelism` when either is set) via `buffer_unordered`, - sharing `Arc>` + sharing one `Arc`. Parallelism reaches the page walk + because each part's tap entries own their own walk state + (`benches/bootstrap_pump.rs` measures it) - `pg_control` parts run as hard barrier after every data part drains — `for key in &control_parts` single-task loop. Multiple control parts is unusual (wal-g emits exactly one) but loop handles it +Part bodies spool to scratch before decode (`spool_backup_part`) so the GET +completes at network speed. Per-entry tap sinks removed the shared-lock +stall the spool was also covering, but it stays: the pipeline downstream +backpressures on ClickHouse by design, and a live body held for however +long CH is the limiter exceeds walrus's 60 s request cap. Costs +`parallelism × part_size` of scratch, compressed + V1 constraint: delta chains error out. Incremented files need disk-resident base to overlay onto via wal-rus's `apply_increment_in_place`, but `Tap` entries never land on disk to be @@ -328,18 +353,37 @@ in which case body drops unread — `PageWalkSink::begin` does this for `pg_control` etc that arrive at user-heap-looking paths or for files whose path does not parse as `base//` -Stats recovery: orchestrator holds two `Arc` clones to same -`Mutex>` — one typed for stats teardown & -one erased (`Arc>`) for source call. `Mutex::into_inner` does not exist (unsized inner); `Arc::try_unwrap` -on typed clone after source returns recovers both inner sinks for -stats reporting +Stats are atomics on `Arc` handles the orchestrator keeps beside the +sink (`BootstrapProgress { pump, page_walk }`, `DiskLanderStats`), so the +sink never has to come back out of the source and `/metrics` reads the +counters while the pump runs. Dropping the last sink `Arc` closes the +tuple-channel sender, which is how a concurrent drain sees EOF ## PageWalkSink -`src/backup_page_walk.rs`. 2A initial-load: Tap user-heap file bodies, -accumulate 8 KiB at a time, walk each full page's `ItemIdData` slots, -decode live tuples through same heap decoder WAL hot path uses +`src/backup_page_walk.rs`. 2A initial-load: `PageWalkSink::begin` +resolves the segment's descriptor and hands back a `PageWalkEntry` that +owns the walk. The entry accumulates body bytes into a `SLAB_BYTES` +slab, walks every complete page in place out of it, carries only a +sub-page remainder into the next slab, and decodes live tuples through +the same heap decoder the WAL hot path uses. + +Two slabs ping-pong per entry: the walk runs on `spawn_blocking` (pure +CPU over a byte slice, no async in it) with at most one in flight, so +the tar reader refills one slab while the other decodes and the async +runtime keeps its workers for the drain stage. Walking in place is what +removes the per-page 8 KiB allocation plus buffer memmove a +`drain(..PAGE_BYTES)` per page costs. + +Per-page counters accumulate in a plain `PageWalkTally` and publish into +the atomic `PageWalkStats` once per slab: a `fetch_add` per tuple, on a +line the reader task also touches, costs more than the decode it counts. + +`begin` also applies the mapped-filenode filter when the orchestrator +passes one (`tap_filenodes`): a relation no mapping routes returns +`Skip`, so its bytes drain off the wire and its pages never decode. +`skip_initial` in the drain stays the authority; the set is a superset +filter, not a second source of truth `heap_decoder::decode_block_data` is exposed as `pub(crate)` for this consumer. On-disk tuple shape carries full `HeapTupleHeaderData` (23 @@ -361,10 +405,12 @@ decoder, exercised from two callers failures bump `tuples_skipped_truncated` so a single torn page does not abort whole bootstrap -`BackfillTuple { rfn, xid, source_lsn, columns }` ships over bounded -mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP`) to orchestrator's drain task. -`source_lsn` is `StartInfo::start_lsn` for every emitted row — every -backfill row tags identically +`BackfillTuple { rfn, xid, source_lsn, columns }` ships in slabs over a +bounded mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP` slabs) to the +orchestrator's drain task. One channel hop per walk slab, not per +tuple; `SLAB_BYTES × CAP` bounds the heap payload in flight per +concurrent segment. `source_lsn` is `StartInfo::start_lsn` for every +emitted row — every backfill row tags identically V1 limits: @@ -391,6 +437,12 @@ V1 limits: - **No 2C CH-side COPY load.** PageWalkSink (2A) is the sole initial-load path; see [Why not 2C](#why-not-2c-ch-side-copy-load) below +`pipeline::bootstrap::drain` coalesces routed rows into +`BatcherMsg::Rows` on the same row/byte dual trigger the streaming +decode pool uses, so a walk slab costs one batcher hop rather than one +per row. The open seq's buffer flushes at an rfn flip before +`placed(prev_seq, rows)` publishes its expected count. + Rfn contiguity buys seq economy, not correctness: `PageWalkSink` emits all rows for one rfn contiguously before moving on, so `pipeline::bootstrap::drain` can synthesize one ack-collector seq per @@ -439,7 +491,7 @@ is the only shape with bounded memory at scale - `seed_in_snapshot(client) -> CatalogMap` — REPEATABLE READ wrapper around `seed_catalog_from_source`. Always COMMITs (read-only xact; commit-vs-rollback is purely about releasing snapshot) -- `spawn_greenfield_bootstrap(cfg, source, catalog_map) -> (mpsc::Receiver, JoinHandle>)` — +- `spawn_greenfield_bootstrap(cfg, source, catalog_map) -> (mpsc::Receiver>, JoinHandle>)` — streaming primitive. Caller drains concurrently with source pump; bounded channel backpressures pump against drain rate, so memory is bounded by `BOOTSTRAP_TUPLE_CHANNEL_CAP`, not source tuple count. @@ -561,6 +613,27 @@ Synchronous `pg_ctl` and `psql` commands run inside wait. `--shadow-socket-dir` and `--shadow-port` configure shadow listener used later by `ShadowCatalog` +## Stage attribution + +`BootstrapProgress` carries the pump's and walk's atomic counters; a ticker +publishes them into `/metrics` while the pump runs and the status loop keeps +rendering their final values afterwards, so a slow initial load stays +attributable after the fact. One `bootstrap stage timings` log line at the +end carries the same numbers: + +- `walshadow_bootstrap_bytes_tapped_total`, + `walshadow_bootstrap_pages_walked_total`, + `walshadow_bootstrap_tuples_emitted_total`, + `walshadow_bootstrap_files_walked_total` — what moved +- `walshadow_bootstrap_files_skipped_unmapped_total` — segments declined at + `begin` because no mapping routes them +- `walshadow_bootstrap_decode_seconds_total` — CPU in the page walk +- `walshadow_bootstrap_channel_block_seconds_total` — the walk waiting on a + free tuple-channel slot, ie emitter drain time seen by the walk +- `walshadow_bootstrap_tap_seconds_total` — time inside the tap entry: + page framing, decode, channel send. Against `decode_seconds` this is + the tap's own overhead + ## Cross-links - [shadow.md](shadow.md) — handoff target. Shadow lifecycle, standby diff --git a/plans/decoder.md b/plans/decoder.md index 3e32e86f..62a7fe92 100644 --- a/plans/decoder.md +++ b/plans/decoder.md @@ -373,7 +373,7 @@ captured fixtures across PG 16/17/18. `tests/classify_fixture.rs` infra snapshots MULTI_INSERT fixtures; the same fixture gap covers `XACT_XINFO_HAS_SUBXACTS` layout. Drift in either record's per-major shape would surface as silent decode mismatch on exactly one major. -Tracked in [future/parked.md](future/parked.md). Structural cousins — +Tracked in [future/risks.md](future/risks.md). Structural cousins — `xl_heap_*` headers, `xl_heap_multi_insert` field order, `attmissingval` encoding — are stable per WAL alignment memory note + direct catalog cross-check against PG 16/17/18 source, absent diff --git a/plans/emitter.md b/plans/emitter.md index 5b5df1d3..4f764cdf 100644 --- a/plans/emitter.md +++ b/plans/emitter.md @@ -4,12 +4,12 @@ CH-side ingest is a parallel decode+insert pipeline (`src/pipeline/`): ```text pump -> QueueingRecordSink -> reorder (plan -> execute) -> [decode x M] - -> InsertBatcher -> [inserter x N] -> ClickHouse - \-> ack collector -> emitter_ack_lsn + -> InsertBatcher -> [resolve x W] -> [inserter x N] -> ClickHouse + \-> ack collector -> emitter_ack_lsn ``` Pipeline stages live in `src/pipeline/{reorder,planner,plan_spool, -decode,batcher,inserter,ack,tail,mod}.rs`; encoding primitives +decode,batcher,resolver,inserter,ack,tail,mod}.rs`; encoding primitives (`EmitterConfig`, `TableEncoder`, `TablePlan`, `ColumnBuf`, value encode, `EmitterStats`) in `src/ch_emitter.rs`; DDL in `src/ch_ddl.rs`; PG → CH type mapping in @@ -187,6 +187,21 @@ the rows' admission/value permits, dropped post-insert-ack): 4-table xacts coalesce into one MergeTree part per window instead of one per xact +### Resolver pool — `pipeline/resolver.rs`, ×W + +W = the bridge's pool width, since that is how many requests the shadow +answers at once. Pops sealed `InsertBatch`es, answers their oracle +columns, and pushes `(batch, local, block)` onto a queue one deep per +inserter. Overlaps `T_oracle` with `T_ch` — sequential stages cost +`T_oracle + T_ch / N`, these cost `max(T_oracle / W, T_ch / N)`. Costs +one extra resident batch per inserter + +A batch with no oracle column crosses without touching the bridge, and +so does one whose oracle columns the daemon can answer itself: cells +already rendered locally (PostGIS WKT via `render_ext_columns`) against +a `String` target are the bytes ClickHouse wants, and PG would hand them +straight back. `oracle_local_columns` counts those + ### Inserter pool — `pipeline/inserter.rs`, ×N N `AsyncClient` connections. CH Cloud INSERT cost is mostly RTT + @@ -199,9 +214,9 @@ owned slabs (`TypeAst` cache keyed on `(table, schema_epoch)`; runs one `send_query` + `send_data` + `send_data_end` + drain-to-`EndOfStream` -A batch holding oracle columns resolves them first: one bridge request -before the query opens, spliced into the same block -([oracle.md](oracle.md)) +Oracle columns arrive already answered: the resolver stage hands over a +block to splice ([oracle.md](oracle.md)), so an inserter's ClickHouse +connection is never idle for a bridge round trip Durability invariant: `ack.acked(per_seq)` fires **only after** the drain returns. Until then a connection drop replays the still-owned diff --git a/plans/future/INDEX.md b/plans/future/INDEX.md index 46797e30..f1e55350 100644 --- a/plans/future/INDEX.md +++ b/plans/future/INDEX.md @@ -20,7 +20,8 @@ rationale under `plans/` only when code cannot express it * [pinned_ddl_baseline.md](pinned_ddl_baseline.md) — schema-event outcome must be a function of config + baseline, not cache warmth: CH-existence / persisted-baseline options for cross-restart consistency, drop detection across downtime, opt-in mapping vs republish * [coverage100.md](coverage100.md) — drive `cargo llvm-cov` line coverage toward 100%: tiered work list (pure units → fixtures → live e2e → hard tail) * [FUZZ.md](FUZZ.md) — continuous coverage-guided fuzzing (cargo-fuzz/libFuzzer) across wal-rus + walshadow + clickhouse-c-rs: tiered targets, round-trip/differential oracles, C-boundary ASan, unattended-VM supervisor +* [perf_regression.md](perf_regression.md) — performance measurement off CI machines: no wall-clock assertions in the suite, own workflow on dedicated hardware, `bench/` initial-load shape, disposable `benches/` microbenches, regression bands * [pipeline_backpressure_and_scaling.md](pipeline_backpressure_and_scaling.md) — parallel decode+insert pipeline: WAL-pump backpressure via wire/record split, decode/insert scaling (bootstrap Option B, hot-table sharding, N/M sizing); pipeline substrate in [emitter.md](../emitter.md) * [dependencies.md](dependencies.md) — crates.io replacement candidates for generic object storage, MPMC, retry, throttling, and metrics code * [risks.md](risks.md) — measurement-deferred risks and open questions -* [parked.md](parked.md) — small operational polish + cross-major fixtures + skipped-test drive +* [parked.md](parked.md) — small operational polish + walsender hardening (TLS/SCRAM, `hot_standby_feedback`) diff --git a/plans/future/coverage100.md b/plans/future/coverage100.md index 00776da1..f0b62200 100644 --- a/plans/future/coverage100.md +++ b/plans/future/coverage100.md @@ -97,6 +97,12 @@ Use existing in-proc and fixture paths. Extend Extend existing `WALSHADOW_USE_LOCAL` e2e tests. Keep live-state cases in existing harnesses rather than adding binaries +Skip gates here are runtime, not `#[ignore]`: a suite checks for `initdb` / +`pg_basebackup` / `clickhouse` on PATH and returns early when absent, so a +job missing one reports the suite as passing while its lines stay uncovered +for no visible reason. Read durations, a live suite that "passes" in ~0 s +did not run + - `shadow_catalog.rs::seed_from_source` and dependent closures: drive via `tests/catalog_seed.rs` / `tests/shadow_catalog.rs` - `ch_emitter.rs::reconnect`: explicit CH drop mid-stream diff --git a/plans/future/ddl_fuzz.md b/plans/future/ddl_fuzz.md index d0738d26..4fe67f1a 100644 --- a/plans/future/ddl_fuzz.md +++ b/plans/future/ddl_fuzz.md @@ -55,6 +55,8 @@ limits from new interactions and gives shrinker stable endpoints | drop PK constraint, drop its column | CH still uses column in `ORDER BY`; CH DROP COLUMN fails after any earlier xact effects | preflight whole plan against target key | | add PK to table created keyless | CH retains `ORDER BY (_lsn)`; updates do not collapse by new PK | rebuild key or keep case unsupported | | `DROP TABLE; CREATE TABLE` with same name under each drop strategy | `retain` / `warn` preserve old rows and `CREATE IF NOT EXISTS` no-ops; `drop` should round-trip | encode policy-specific lifecycle oracle | +| `DROP TABLE s.t CASCADE` with a dependent view, FK child or partition child | basic path emits the drop via `SchemaEvent` + `DrainEntry::Catalog`; dependent relations get no event of their own | decide whether cascaded relations follow the drop strategy or are left to drift | +| `DROP TABLE s.t RESTRICT` refused by source, then DML on `s.t` | source rejects, so no WAL and no event; mapping must stay intact | assert a refused DDL leaves no destination effect | | `CREATE UNLOGGED TABLE; INSERT ...` | catalog may create CH table; user DML has no durable WAL | reject mapping or mark no-row policy explicitly | | `ALTER TABLE ... SET UNLOGGED`, mutate, `SET LOGGED` | unlogged interval disappears; stale CH rows can survive conversion | reject persistence transition | | attach populated partition, then write through parent | heap WAL names leaf; pinned parent target receives no fan-in and attach emits no row backfill | define partition routing/backfill semantics | diff --git a/plans/future/parked.md b/plans/future/parked.md index b9e01633..7ab70514 100644 --- a/plans/future/parked.md +++ b/plans/future/parked.md @@ -1,7 +1,12 @@ # parked — small operational debt and follow-up polish -Operational debt collected from retros plus follow-ups from the -allocation-audit pass. One-line per item +Operational debt collected from retros, one line per item. Items that fit +a subject doc live there instead: perf and profiling work in +[perf_regression.md](perf_regression.md), coverage and skip-gate hazards in +[coverage100.md](coverage100.md), decoder-fidelity risks in +[risks.md](risks.md), config knobs in +[runtime_config_from_pg.md](runtime_config_from_pg.md), DDL transition +corpus in [ddl_fuzz.md](ddl_fuzz.md) ## v1.0 operational polish @@ -13,49 +18,6 @@ allocation-audit pass. One-line per item spawn own CH (~5 s × N startup). Total CI cost ~25 s of unique boot time. Flag if test count doubles -## Cross-major fixture pinning - -* **MULTI_INSERT + xl_xact_commit fixtures against PG 16/17/18 via - `tests/classify_fixture.rs`.** Cross-major drift in tail-walk - semantics would surface as silent decoder mismatch under one - specific major. cross-major snapshot fixtures called for snapshot - fixtures across majors - -## Drive currently-skipped tests - -Acceptance tests ship with runtime skip-gates checking for `initdb` / -`pg_basebackup` / `clickhouse` on `PATH`; *not* `#[ignore]`. Each -needs source PG + CH + (usually) basebackup-cloned shadow. Drive in -CI when those binaries reliably present: - -* `kill_restart` -* `pgbench_acceptance` -* `bootstrap_direct_ch` -* `bootstrap_object_store_ch` -* `truncate` -* `subxact` -* `copy_into` -* `add_column_default` - -Each is a one-line un-skip + observation of which fixture path -needs a kick. Acceptance items §1 (pgbench), §5 (kill-restart) -remain unverified against live topology until driven - -## Zero-copy follow-ups - -* **Criterion benchmark.** Allocation-count + RSS measurement - post-hoc; land `benches/` crate when measurement contested. - Targets predicted RSS drop (≈200 MB → ≈0 for 100k-record - heap-INSERT segment) + 1.5-3× decode throughput from dropped - allocator pressure -* **`XLogRecord.blocks` smallvec.** Records average 0-2 blocks; - `SmallVec<[_; 2]>` keeps common case stack-resident. Allocation - polish below byte-traffic wins from Cow -* **Header-walk single-pass merge.** `record.blocks` walk runs - twice (once for IDs, once for payloads) in wal-rus parser. Merge - into single pass since IDs arrive in order. Leftover from wal-g - port - ## Walsender hardening * **TLS / SCRAM auth.** Trust-over-loopback only today. @@ -68,17 +30,3 @@ remain unverified against live topology until driven * **Walsender keepalive-timeout unit test.** Indirectly covered by libpq + PG-walreceiver round-trips in `walsender_pg18_walreceiver`; explicit unit test is polish - -## Decoder follow-ups - -* **Subxact `XACT_XINFO_HAS_INVALS` ordering verification fixture.** - Capture commit record from PG with all `xinfo` bits set; prove - walk doesn't drift under out-of-the-way ordering on some major. - Subxact retro flagged this -* **TRUNCATE strategy knob.** v1 emits single `TRUNCATE TABLE ` - per relation. Per-table `truncate_strategy = "passthrough" | - "ignore"` knob once downstream consumer asks. Defer-until-asked -* **DROP TABLE propagation polish.** Basic path runs via - `SchemaEvent` + `DrainEntry::Catalog` channel. Corner cases - (CASCADE, RESTRICT, dependent objects) need pinning against - fixture matrix diff --git a/plans/future/perf_regression.md b/plans/future/perf_regression.md new file mode 100644 index 00000000..3599b87d --- /dev/null +++ b/plans/future/perf_regression.md @@ -0,0 +1,245 @@ +# perf_regression — measurement off CI machines + +Performance claims get their own workflow on hardware nobody else is +using. `cargo nextest` (and the coverage job that wraps it) proves +behavior; it never proves speed. This doc sets that boundary, then +specifies the workload `bench/` needs to cover initial load, which is the +largest unmeasured stage + +## Why CI cannot hold a ratio + +A GitHub-hosted runner is 4 shared vCPUs running the whole suite at +`num-cpus` width, so a timing assertion measures scheduler luck. The +coverage job is worse: instrumentation adds counter writes to every +region, which lands unevenly across a pooled path and moves the ratio +with no code change + +Concretely, the bridge worker pool compares 4 concurrent `ENCODE_NATIVE` +over 4 sockets against the same 4 over 1 socket, best of three a side, and +asserts speedup above 1.4. It scores ~2.4x on a 16-core box, ~1.7x on a +4-vCPU runner and 1.24x (90.9ms vs 112.6ms) under `cargo llvm-cov` on a +runner also executing ~1200 other tests. Cores cap the win, so the bar can +only be set for a machine that the suite does not get to pick + +Rule: no assertion in the test suite may compare wall clock against a +constant or against another wall clock. Printing timings is fine, the WAL +pump microbenchmark test does exactly that and asserts only record counts + +## What the suite keeps + +Splitting a perf test means keeping its correctness half, which is most of +it. For the bridge pool that is: pool width equals the requested worker +count and is visible in `stats.pool_size`, every slot completed its own +HELLO, `info()` is populated, each concurrent request returns one offset +per row, and catalog `scan` stays pinned to slot 0 whatever the width. +None of those touch a clock + +The ratio moves to the oracle roundtrip stage, whose job already is +pricing one batch alone and with the pool saturated. Report core count +beside the ratio, since that is the term that sets the ceiling + +## bench/ is durable, benches/ is not + +Two homes, different lifetimes: + +* `bench/` is a deployed workload engine: a crate with shapes, an + instrument split (`count_all` cadence for throughput, `count_id` probes + for latency), a `Destination` abstraction covering ClickHouse and a PG + standby, plus the EC2 harness that stands up a comparable box. Shapes + are cross-engine, so they survive any internal refactor +* `benches/*.rs` are task-scoped microbenches. Each exists to settle one + contested optimisation (pump-vs-drain CPU split, oracle framing passes) + and stops earning its keep once that question closes. Expect the + directory to churn and to empty + +So the perf workflow keys off `bench/` shapes and never off a `[[bench]]` +target name. Deleting a microbench must stay a file delete plus a manifest +stanza: `docker/Dockerfile.bench` already copies `src` and `bench` without +`benches` and builds `-p walshadow-bench`, so the deployed driver does not +see them. A microbench that does stay must answer `--list` with +`: benchmark`, else `nextest --all-targets` fails listing + +## Finding the next bottleneck + +Gating answers "did this get worse". Optimisation work asks the other +question, "what owns the time", and that needs a profiler plus one +microbench per contested claim, on the same dedicated box + +On-CPU capture: `bench/ec2/profile.sh [secs]` starts a capture +beside a run and teardown copies the result into the node folder. The +release profile carries `debug = "line-tables-only"`, so `perf annotate` +and `--sort=srcline` attribute samples to source lines at no runtime cost + +Criterion earns its keep where a claim is a per-call cost over a stable +input: statistical comparison, saved baselines, outlier detection. Where +the claim is a staged pipeline needing allocator choice, thread counts and +synthetic segments, the flag-driven `harness = false` shape stays. Either +way the allocator must match the daemon's (mimalloc): the walk allocates +decoded values on one thread and the drain frees them on another, and +glibc's shared arena serializes on that pattern hard enough to invert the +conclusion, reporting a parallel pump as slower than the serial one + +Allocation count and peak RSS are a separate instrument from wall clock, +answered post hoc by a counting allocator wrapper plus RSS sampling, per +stage. The zero-copy framing pass predicts a 100k-record heap-INSERT +segment dropping ~200 MB of allocator traffic to ~0, worth 1.5-3x decode +throughput off lost allocator pressure. Unmeasured + +Candidates already named, each wanting a number before code: + +* **CRC32C rewrite parallelism.** The filter rewrites every kept record's + CRC on one thread at ~1 ns/byte, so 1 GB/s of WAL saturates a core. + Records are independent post-classification, see [risks.md](risks.md) +* **`XLogRecord.blocks` allocation.** `Vec` per record + where records average 0-2 blocks; `SmallVec<[_; 2]>` keeps the common + case stack-resident. Ranked below byte-traffic wins +* **Block-header walk.** The parser walks blocks twice, once pushing + headers and once attaching image/data slices. Not redundant: a record + body carries every block header before any block payload, which is why + PostgreSQL's own `DecodeXLogRecord` splits the same way. Any win here is + pre-sizing the vector, not merging the passes + +## Workflow shape + +Trigger on `workflow_dispatch` plus a schedule, never on pull_request. +Runs either on a single-tenant self-hosted runner or by driving +`bench/ec2/stack.sh` (`up `, `bench run `, `down`), which +already pins instance type and AZ and writes `provenance.txt` (setup, +instance type, AZ, commit) beside the results. Publish numbers as +artifacts; do not fail a PR on them + +## Shape: initial load + +`bench/` covers steady-state only. Initial load is where BASE_BACKUP wire +throughput, source-PG read cost and ClickHouse insert saturation meet, and +it has no shape + +### Two triggers, different cost + +Greenfield bootstrap (`--bootstrap-mode=direct|object_store`) runs only +against an empty shadow data dir with no completion marker. Triggering it +means wiping the data volume and restarting the container, lifecycle +control the bench engine does not have by design + +Per-table backup backfill (`initial_load='base_backup'|'object_store'`) +drives the same `PageWalkSink` → gate → bootstrap-drain path on a live +daemon. Trigger is an INSERT into `config_table` on the source plus +`pg_switch_wal()`, nothing else + +Both cover the page walk, the backup sink and source, and the bootstrap +drain. Per-table needs no restart and no SSH, so it carries the primary +workload; greenfield rides a second shape behind a reset hook + +### Assumptions the engine breaks + +* `run()` truncates the source table unconditionally. An initial-load + dataset *is* pre-existing source state, so the clean-slate preamble goes + per shape: clear the destination, never the source +* `Throughput::from_curve`'s `max_backlog` term is `achieved_rate * at`, + meaningful only for an interval-driven producer. Against a static source + it is fabricated, so suppress it. `all_visible_at` and `peak_rate` stay + valid +* The latency instrument has no meaning here, no commit instant exists. + `count_id` probes, probe slot states and the probe clock all drop out. + Throughput instrument only +* Seed into `demo.bl_`, never the table other shapes truncate. An + auto-create namespace rule already covers new tables there + +### `--bench initial-load` + +1. **Seed** `CREATE TABLE demo.bl_`, fill server-side via + `INSERT … SELECT FROM generate_series`, rows and width from flags. + Report seed wall time and `pg_total_relation_size` apart from the + measurement. Relation size is the MiB/s denominator +2. **Trigger** INSERT into `config_table(namespace, relname, replicate, + initial_load)`, then `pg_switch_wal()`. `t0` is that commit on the + driver's own monotonic clock, no cross-host coordination +3. **Watch** reuse the `count_all` sampler at `count_interval_ms`. + Completion predicate `count_all >= N` is exact while the source stays + static, no FINAL, matching the no-FINAL discipline elsewhere +4. **Report** elapsed, rows/s, MiB/s over heap bytes, peak sampled rate, + plus stage attribution when a metrics endpoint is configured + +Per-run reset is a fresh table name, not a restart: the spill-dir backfill +ledger marks a qname done and no later boot re-runs it. A fresh table per +run also grows cluster size, which is how `base_backup`'s cluster-sized +bandwidth cost becomes visible + +Mode sweep (`copy` vs `base_backup` vs `object_store`) is one flag on the +same shape, under one instrument + +### Concurrent-write variant + +`--concurrent-rate` inserts during a pass, exercising walk/live-stream +overlap and the staging swap. The completion predicate must then become +`count(distinct id) == N` or FINAL: overshoot past N is legitimate there, +dedup absorbs it at merge + +### Stage attribution + +`--metrics-url` scrapes the daemon's metrics endpoint on the count cadence +(GET; the existing ClickHouse HTTP client is POST-only, so this needs a +sibling): + +* `walshadow_config_backfills_pending{mode="…"}`, the daemon-side + start/stop bracket for a pass +* `walshadow_bootstrap_decode_seconds_total`, + `walshadow_bootstrap_tap_seconds_total`, + `walshadow_bootstrap_channel_block_seconds_total` +* `walshadow_inserter_ch_seconds_total`, + `walshadow_inserter_encode_seconds_total` +* `walshadow_process_cpu_seconds_total` + +Answers which of tap, decode, channel block or ClickHouse insert owns wall +clock at real scale. Unset skips it, so the same shape still runs against +PeerDB's snapshot and a standby's `pg_basebackup`, giving a cross-engine +initial-load comparison + +### Daemon prerequisites + +* The per-table backfill builds its walk sink without stats and hands the + source a throwaway `PumpStats`, discarding every stage counter. Thread a + shared progress handle through the pass context and publish it (same + series, or `phase=` labelled) or the restart-free workload stays + wall-clock only +* The bootstrap-phase ticker publishes bootstrap fields over + `MetricsSnapshot::default()`, so every other series, uptime included, + reads 0 for the bootstrap duration then jumps when the status loop takes + over. That reads as a counter reset to any scraper and rules out uptime + as a `t0` anchor. Merge into the live snapshot instead of replacing it + +### `--bench bootstrap` (greenfield) + +Keep out of `--suite`: the steady-state pass is self-contained, this is +not. Add `stack.sh bench bootstrap ` alongside `run`/`fetch`, +sequencing seed → wipe volume + redeploy → one shape → fetch + +Give the shape `--reset-cmd ''` so the bench owns `t0`: run the +command, anchor at the first metrics scrape that answers *after* a +refusal. A first scrape that already answers means daemon start time is +unknown, so report that instead of anchoring wrong, the same rule the +sustained shape applies when rows go missing. One flag covers local +compose and EC2 SSH without the engine knowing either; SSH stays out of +the engine and the local driver shares the flag + +### Relation to the pump microbench + +Not redundant, and not a substitute. The microbench isolates pump-vs-drain +CPU on one host with in-memory tars. This workload measures wire +throughput, source-PG cost and destination insert saturation at scale, and +it outlives the optimisation that motivated the microbench + +## Calling a regression + +Numbers from a shared box are unusable, and numbers from a dedicated box +are only usable with a band. Establish the band before gating anything: +repeat the same shape on the same commit until the spread is known, and +express thresholds in terms of that spread, not a guessed percent + +* Report median and best-of together. Best-of hides contention, median + hides the tail; a move in one only is a machine story +* Require a regression to reproduce on a second pass before it counts +* Key stored baselines by (setup, instance type, shape flags, commit). + A baseline from another instance type is not a baseline +* Publish first, gate later. A perf gate that fires on variance gets + disabled, which is worse than no gate diff --git a/plans/future/risks.md b/plans/future/risks.md index bad7e928..2f5dfc1c 100644 --- a/plans/future/risks.md +++ b/plans/future/risks.md @@ -66,6 +66,32 @@ not from source. Source on tzdata 2023d, shadow's host on tzdata 2024a, divergence on rare-but-real timestamp tzname output. Mitigation: pin tzdata version at deploy time. Not enforced +## Cross-major commit-record tail walk + +A commit record's tail is variable-shape: `xl_xact_commit` appends +subxact array, dropped-stats items, relfilelocators, invalidation +messages, twophase gid and origin, each behind its `xl_xact_xinfo` bit. +Reaching a later field means skipping every earlier one at the right +width, so a wrong width reads a valid-looking record wrong and CRC +catches nothing: the bytes are intact, the interpretation is not + +The shape does move. `xl_xact_stats_item` went 12 → 16 bytes in PG 18 +(PostgreSQL commit `b14e9ce7d55`, `PgStat_HashKey.objid` widened to 8 +bytes carried as two `uint32`), and the `SysCacheIdentifier` values the +invalidation walk matches on shifted 35/36 → 37/38 in the same major. +[`src/decode/wal_xact.rs`](../../src/decode/wal_xact.rs) branches on +`page_magic` for both. `MULTI_INSERT` carries the same class of exposure, +its per-tuple walk reads header flags rather than a fixed stride + +Risk is a per-major branch that no test pins, so the next widening or id +shift is silent on exactly one major. Mitigation is fixture pinning +through [`tests/classify_fixture.rs`](../../tests/classify_fixture.rs): +capture a commit record with every `xinfo` bit set (subxacts, dropped +stats, relfilelocators, `XACT_XINFO_HAS_INVALS`, origin) plus a +`MULTI_INSERT` batch, on 16 / 17 / 18, and assert each major's walk lands +on the same fields. Capture is already scripted per major; the snapshots +are what is missing + ## Path A CRC at >1 GB/s WAL Filter rewrites every kept record's CRC32C; today single-threaded. @@ -77,8 +103,8 @@ classification). Defer thread pool until measurement demands. overview.md pitfall #8 flagged this Zero-copy framing already cut allocator pressure off the hot path; -CRC is the next bottleneck if `criterion` benchmarks land and -surface it. Bench is itself deferred, see [parked.md](parked.md) +CRC is the next bottleneck once a bench surfaces it, see +[perf_regression.md](perf_regression.md) ## PG fork temptation diff --git a/plans/future/runtime_config_from_pg.md b/plans/future/runtime_config_from_pg.md index fcfccec1..dd47bad5 100644 --- a/plans/future/runtime_config_from_pg.md +++ b/plans/future/runtime_config_from_pg.md @@ -157,6 +157,7 @@ behind it, distinct from knobs resolved by [`src/config.rs`](../../src/config.rs |---|---|---|---| | `engine` / `order_by` | table, namespace | text | fixed in `ch_ddl` (engine hardcoded `ReplacingMergeTree`, order_by derived from PK/replica-identity index); shape change, needs rfn drain + `TablePlan` rebuild | | `exclude` | column | bool | `ColumnMapping` has no such field; drops a column from projection + future DDL; shape change | +| `truncate_strategy` | table, namespace | text | `passthrough` (today's single `TRUNCATE TABLE ` per relation) or `ignore`; non-shape key, defer until a downstream consumer asks | | `ch_settings` | global, namespace, table | jsonb | applied to INSERT/CREATE TABLE, merged narrow-wins | | `sample_rate` | (TOML only) | float | emitter row-drop sampling for debug, distinct from the `--validate` oracle sampler in `src/oracle.rs` | | `signal_prefix` | (TOML only) | text | which `pg_logical_emit_message` prefix to scan | diff --git a/plans/oracle.md b/plans/oracle.md index af637b7d..e30af10d 100644 --- a/plans/oracle.md +++ b/plans/oracle.md @@ -20,8 +20,9 @@ interprets array elements, hstore pairs, JSON text, null maps, or nested offsets Resolution runs at sealed-batch granularity, one request per `InsertBatch`, -after the batcher has fixed row order and before the inserter opens its -ClickHouse query. It is not best-effort: one Datum the worker cannot convert +in the resolver stage between batcher and inserter pool +([emitter.md](emitter.md)) so a round trip overlaps other inserters' queries +rather than idling their connections. It is not best-effort: one Datum the worker cannot convert fails the whole batch, because a substituted value is one the destination cannot tell from real data @@ -66,8 +67,8 @@ String PostGIS `geography` / `geometry` route to the oracle like any other extension type, but `render_ext_columns` (`src/ops/oracle.rs`) still claims 2-D points in-tree first, matched on `RelAttr.type_name` because their OIDs are dynamic. -A rendered `POINT(x y)` crosses as a literal cell and lands verbatim; anything -it does not claim crosses as on-disk bytes and gets `typoutput`. `ST_AsText` +A rendered `POINT(x y)` lands verbatim; anything it does not claim crosses as +on-disk bytes and gets `typoutput`. `ST_AsText` semantics are what the `String` mapping wants, and `typoutput` would give HEXEWKB instead @@ -124,8 +125,8 @@ all of that passes ## Splicing -The inserter decodes the response once with `clickhouse-c-rs`, then builds the -final block over three kinds of column: +The resolver decodes the response once with `clickhouse-c-rs`, then the +inserter builds the final block over three kinds of column: - local fixed / string / nullable slabs, through `build_leaf` + `build_root` - decoded oracle column trees, through `BlockBuilder::append_column` @@ -137,6 +138,20 @@ expected `TypeAst`. Nothing visits a value. The response block outlives the whole `send_with_retry` loop, so a ClickHouse reconnect resends the same columns rather than asking the oracle again +## Columns the request leaves out + +A column whose every cell is `Literal` or `Default`, against a `String` or +`Nullable(String)` target, is bytes the daemon rendered and ClickHouse wants +unchanged: the worker would wrap each in a `text` and pgch would cast it +straight back. `literal_column` builds those in the resolver, matching +`ws_append_default` on defaults — NULL under `Nullable`, empty string +otherwise. Anything wrapping a String (`Array`, `LowCardinality`) is PG's to +parse, and one un-rendered cell keeps its whole column in the request + +Live case is a PostGIS 2-D point column with a `String` target, which is +often a table's only oracle column — then the batch makes no request at all. +`walshadow_oracle_local_columns_total` counts columns taken this way + ## walshadow PG module Lives at [`pgext/`](../pgext/), built via PGXS. Not an extension: no control @@ -239,6 +254,26 @@ bound to the requested transaction. Losing the oid list loses the lock argument with it, which is why an empty list is only safe on the committed read — there is no tree to misattribute a writer to +## Worker pool + +Each worker serves one request per loop iteration, so one worker means +oracle throughput is one PG backend's conversion rate however wide the +daemon's inserter pool is. `walshadow.bridge_workers` (default 1, ceiling +`WS_MAX_WORKERS`) registers copies: worker 0 keeps the bare +`walshadow.socket_path`, worker `i` listens on `socket_path.i`. On a +daemon-owned shadow `--bridge-workers` writes that GUC, sizes the daemon's +socket pool to match, and with it the resolver pool + +`ENCODE_NATIVE` and `REPLAY_LSN` are stateless and read-only, so the pool +round-robins them. `HELLO` and `SCAN` pin to worker 0: a scan answers off +a replay position it reports back, and the first `HELLO` establishes the +identity every other worker is then checked against, so a build mismatch +across the pool fails closed at connect rather than mid-read + +`Bridge::connect_pooled` requires every configured slot to answer. A pool +short of its configured width is a half-started shadow, and running +narrower would hide it + ## Failure semantics Daemon requires worker at startup. `--bridge-socket` defaults to @@ -265,7 +300,13 @@ make a value depend on worker uptime Counters: `walshadow_oracle_{blocks,rows,cells}_total`, `walshadow_oracle_conversion_errors_total`, `walshadow_oracle_errors_total`. Native volume counts once, on the bridge, as -`walshadow_bridge_native_bytes_total` +`walshadow_bridge_native_bytes_total`. +`walshadow_bridge_lock_wait_seconds_total{op}` against +`walshadow_bridge_service_seconds_total{op}` is what says whether the +worker or the socket in front of it is the limiter, and +`walshadow_oracle_resolve_seconds_total` against +`walshadow_inserter_ch_seconds_total` which of the two overlapped stages +is ## Backpressure @@ -281,7 +322,8 @@ above `inline_value_max` so a toast value the pipeline admits always frames Removing per-row resolution from the decode pool removes every bridge await from decode workers: their throughput and ordering no longer depend on bridge -RTT +RTT. The resolver stage holds one extra batch per inserter beyond that, which +is what a queue deep enough to keep every inserter fed costs ## Pinning shadow locale diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index d049ce21..eb798b4f 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -27,7 +27,7 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{Context, Result}; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc; use tokio::task::JoinHandle; use tokio_postgres::Client; use walrus::pg::walparser::{Oid, RelFileNode}; @@ -38,10 +38,19 @@ use crate::backfill::backup_page_walk::{ use crate::backfill::backup_sink::{ CatalogFilenodes, DiskLanderSink, DiskLanderStats, MultiplexSink, }; -use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, StartInfo}; +use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, PumpStats, StartInfo}; use crate::decode::decoder_sink::TupleObserver; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; +/// Live counter handles, readable while the pump runs. The pump publishes +/// into these, so a caller ticks `/metrics` off them instead of waiting for +/// [`BootstrapOutcome`] +#[derive(Debug, Clone, Default)] +pub struct BootstrapProgress { + pub pump: Arc, + pub page_walk: Arc, +} + #[derive(Debug, Clone)] pub struct BootstrapConfig { /// Orchestrator pre-creates if missing. `Shadow::start` requires it @@ -51,6 +60,10 @@ pub struct BootstrapConfig { /// `rel_node < 16384` bootstrap rule misses. Empty in greenfield where /// seed runs after bootstrap pub catalog_filenodes: CatalogFilenodes, + /// Filenodes worth decoding: mapped relations plus their TOAST heaps. + /// `None` taps every seeded relation + pub tap_filenodes: Option>>, + pub progress: BootstrapProgress, } impl BootstrapConfig { @@ -58,6 +71,8 @@ impl BootstrapConfig { Self { shadow_data_dir, catalog_filenodes: CatalogFilenodes::new(), + tap_filenodes: None, + progress: BootstrapProgress::default(), } } @@ -65,14 +80,65 @@ impl BootstrapConfig { self.catalog_filenodes = c; self } + + /// Decline every filenode outside `set` at `begin`, so unmapped + /// relations drain off the wire without a page decode + pub fn with_tap_filenodes(mut self, set: Arc>) -> Self { + self.tap_filenodes = Some(set); + self + } } #[derive(Debug, Clone)] pub struct BootstrapOutcome { pub start: StartInfo, pub end: EndInfo, - pub disk: DiskLanderStats, - pub page_walk: PageWalkStats, + pub disk: Arc, + pub page_walk: Arc, + pub pump: Arc, +} + +/// Filenodes worth page-walking: relations `mapped` accepts, plus the TOAST +/// heap of each. `begin` declines everything else, so an unreplicated +/// relation's bytes drain off the wire without a page decode. +/// +/// `None` disables the filter: a mapped relation whose `reltoastrelid` the +/// seed did not resolve would otherwise have its TOAST heap skipped, and its +/// referring rows would fail resolution mid-drain. Fail open, walk everything. +pub fn tap_filenode_set( + catalog: &CatalogMap, + mapped: impl Fn(&RelName) -> bool, +) -> Option> { + use ahash::HashSetExt; + let by_oid: ahash::HashMap = catalog + .descriptors() + .map(|d| (d.oid, (d.rfn.db_node, d.rfn.rel_node))) + .collect(); + let mut set = ahash::HashSet::new(); + for d in catalog.descriptors() { + if !mapped(&d.rel_name) { + continue; + } + set.insert((d.rfn.db_node, d.rfn.rel_node)); + if d.toast_oid == 0 { + continue; + } + match by_oid.get(&d.toast_oid) { + Some(&key) => { + set.insert(key); + } + None => { + tracing::warn!( + target: "walshadow::bootstrap", + relation = %d.rel_name, + toast_oid = d.toast_oid, + "toast heap absent from the catalog seed, walking every relation", + ); + return None; + } + } + } + Some(set) } /// Spawn pump on a tokio task, yield `(rx, handle)` so caller drains @@ -102,12 +168,12 @@ pub fn spawn_greenfield_bootstrap( // count + skip (disabled mode, values NULL/default-filled). store_toast: bool, ) -> ( - mpsc::Receiver, + mpsc::Receiver>, JoinHandle>, ) { // Bounded: PageWalkSink::chunk awaits a free slot, so a slow drain // backpressures the source instead of buffering the whole relation - let (tx, rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (tx, rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); let pump = tokio::spawn(async move { tokio::fs::create_dir_all(&cfg.shadow_data_dir) .await @@ -119,46 +185,31 @@ pub fn spawn_greenfield_bootstrap( })?; let lander = DiskLanderSink::new(cfg.catalog_filenodes); - let page_walk = PageWalkSink::new(catalog_map, tx, store_toast); - let mux = MultiplexSink::new(lander, page_walk); - - // Keep typed Arc beside erased trait-object Arc to recover stats - // after source completes; both point at the same Mutex - let typed: Arc>> = Arc::new(Mutex::new(mux)); - let erased: Arc> = typed.clone(); + let disk = lander.stats.clone(); + let mut page_walk = PageWalkSink::new(catalog_map, tx, store_toast) + .with_stats(cfg.progress.page_walk.clone()); + if let Some(set) = cfg.tap_filenodes.clone() { + page_walk = page_walk.with_tap_filenodes(set); + } + // Counters are atomics on shared handles, so the sink never has to + // come back out of the source + let sink: Arc = Arc::new(MultiplexSink::new(lander, page_walk)); let data_dir = cfg.shadow_data_dir.clone(); let (start, end) = source - .run(data_dir, erased) + .run(data_dir, sink, cfg.progress.pump.clone()) .await .context("bootstrap: source.run")?; - // Source dropped its `erased` clone on return, so `try_unwrap` - // succeeds unless a clone leaked; else read through the Mutex - let outcome = match Arc::try_unwrap(typed) { - Ok(mtx) => { - let mux = mtx.into_inner(); - let (lander, page_walk) = mux.into_inner(); - // Dropping `page_walk` closes the channel sender in `out_tx`, - // so a concurrent drain observes channel-close on next recv - BootstrapOutcome { - start, - end, - disk: lander.stats, - page_walk: page_walk.stats, - } - } - Err(arc) => { - let g = arc.lock().await; - BootstrapOutcome { - start, - end, - disk: g.lander_stats().clone(), - page_walk: PageWalkStats::default(), - } - } - }; - Ok(outcome) + // Dropping the last sink Arc closes the channel sender in `out_tx`, + // so a concurrent drain observes channel-close on next recv + Ok(BootstrapOutcome { + start, + end, + disk, + page_walk: cfg.progress.page_walk.clone(), + pump: cfg.progress.pump.clone(), + }) }); (rx, pump) } @@ -176,8 +227,8 @@ pub async fn run_greenfield_bootstrap( let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); let drain = tokio::spawn(async move { let mut out = Vec::new(); - while let Some(t) = rx.recv().await { - out.push(t); + while let Some(slab) = rx.recv().await { + out.extend(slab); } out }); @@ -202,28 +253,30 @@ pub async fn run_greenfield_bootstrap( /// Final `on_xact_end` after channel-close lets the transitional emitter /// release CH state before the daemon swaps to the shadow-catalog emitter. pub async fn drain_backfill( - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver>, observer: &mut O, ) -> Result { let mut shipped: u64 = 0; let mut last_rfn: Option = None; let mut last_lsn: u64 = 0; - while let Some(tuple) = rx.recv().await { - if let Some(prev) = last_rfn - && prev != tuple.rfn - { - observer.on_xact_end(last_lsn).await.map_err(|e| { - anyhow::anyhow!("bootstrap drain: emitter rejected mid-table xact end: {e}") - })?; + while let Some(slab) = rx.recv().await { + for tuple in slab { + if let Some(prev) = last_rfn + && prev != tuple.rfn + { + observer.on_xact_end(last_lsn).await.map_err(|e| { + anyhow::anyhow!("bootstrap drain: emitter rejected mid-table xact end: {e}") + })?; + } + last_rfn = Some(tuple.rfn); + last_lsn = tuple.source_lsn; + let committed = tuple.into_committed_insert(); + observer + .on_tuple(&committed) + .await + .map_err(|e| anyhow::anyhow!("bootstrap drain: emitter rejected tuple: {e}"))?; + shipped += 1; } - last_rfn = Some(tuple.rfn); - last_lsn = tuple.source_lsn; - let committed = tuple.into_committed_insert(); - observer - .on_tuple(&committed) - .await - .map_err(|e| anyhow::anyhow!("bootstrap drain: emitter rejected tuple: {e}"))?; - shipped += 1; } observer .on_xact_end(last_lsn) @@ -424,27 +477,17 @@ mod tests { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { - { - let mut g = sink.lock().await; - g.start(&self.start).await?; - } - for (i, (meta, body)) in self.files.iter().enumerate() { + sink.start(&self.start).await?; + let target = + crate::backfill::backup_source::PumpTarget::new(data_dir, sink.clone(), stats); + for (meta, body) in self.files.iter() { let mut cur: &[u8] = body; - crate::backfill::backup_source::pump_entry( - &mut cur, - meta, - &data_dir, - &sink, - crate::backfill::backup_source::EntryId(i as u64), - ) - .await?; - } - { - let mut g = sink.lock().await; - g.finish(&self.end).await?; + crate::backfill::backup_source::pump_entry(&mut cur, meta, &target).await?; } + sink.finish(&self.end).await?; Ok((self.start.clone(), self.end.clone())) } } @@ -537,14 +580,14 @@ mod tests { async fn drain_backfill_synthesises_inserts_into_observer() { use crate::decode::decoder_sink::CollectingTupleObserver; - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); let rfn = RelFileNode { spc_node: 1663, db_node: 5, rel_node: 16400, }; for v in 0..3 { - tx.send(BackfillTuple { + tx.send(vec![BackfillTuple { rfn, xid: 100 + v, xmax: 0, @@ -555,7 +598,7 @@ mod tests { columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( v as i32, ))], - }) + }]) .await .unwrap(); } @@ -623,9 +666,9 @@ mod tests { db_node: 5, rel_node: 16400, }; - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); for v in 0..4u32 { - tx.send(BackfillTuple { + tx.send(vec![BackfillTuple { rfn, xid: v, xmax: 0, @@ -636,7 +679,7 @@ mod tests { columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( v as i32, ))], - }) + }]) .await .unwrap(); } @@ -654,7 +697,7 @@ mod tests { async fn drain_backfill_calls_on_xact_end_even_on_empty_channel() { // Sender dropped without a tuple; `on_xact_end` still fires so the // transitional emitter's INSERT cleanup runs unconditionally - let (tx, rx) = mpsc::channel::(64); + let (tx, rx) = mpsc::channel::>(64); drop(tx); let mut obs = CountingObserver::default(); let shipped = drain_backfill(rx, &mut obs).await.unwrap(); diff --git a/src/backfill/backup_backfill.rs b/src/backfill/backup_backfill.rs index fec9f6af..034a7403 100644 --- a/src/backfill/backup_backfill.rs +++ b/src/backfill/backup_backfill.rs @@ -55,7 +55,7 @@ use crate::backfill::backup_page_walk::{ BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTuple, CatalogMap, PageWalkSink, }; use crate::backfill::backup_sentinel::build_lsn_pair; -use crate::backfill::backup_source::{BackupSink, BackupSource}; +use crate::backfill::backup_source::{BackupSink, BackupSource, PumpStats}; use crate::backfill::backup_source_direct::DirectSource; use crate::backfill::backup_source_object_store::ObjectStoreSource; use crate::backfill::spool::{DEFERRED_SPOOL_MEM_MAX, DeferredSpool}; @@ -312,14 +312,15 @@ async fn walk_and_ship( let pg_xact = Arc::new(std::sync::Mutex::new(PgXactAccum::new())); let pg_multixact = Arc::new(std::sync::Mutex::new(PgMultiXactAccum::new())); - let (walk_tx, walk_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); - let (gated_tx, gated_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); - - let sink = PageWalkSink::new(filter.clone(), walk_tx, store_toast) - .with_pg_xact_accum(pg_xact.clone()) - .with_pg_multixact_accum(pg_multixact.clone()) - .with_lsn_overrides(lsn_overrides); - let erased: Arc> = Arc::new(Mutex::new(sink)); + let (walk_tx, walk_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (gated_tx, gated_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); + + let sink: Arc = Arc::new( + PageWalkSink::new(filter.clone(), walk_tx, store_toast) + .with_pg_xact_accum(pg_xact.clone()) + .with_pg_multixact_accum(pg_multixact.clone()) + .with_lsn_overrides(lsn_overrides), + ); // data_dir is never written: PageWalkSink only Taps/Skips let data_dir = ctx.scratch_dir.join("void"); @@ -360,7 +361,7 @@ async fn walk_and_ship( // against a complete pg_xact accum; a failed source drops the sender // and the gate discards them instead let run_res = source - .run(data_dir, erased) + .run(data_dir, sink, Arc::new(PumpStats::default())) .await .context("backup_backfill: source.run"); if run_res.is_ok() { @@ -469,8 +470,8 @@ struct GateStats { /// predates WAL coverage, skipping risks dropping a live row. #[allow(clippy::too_many_arguments)] async fn gate_task( - mut rx: mpsc::Receiver, - tx: mpsc::Sender, + mut rx: mpsc::Receiver>, + tx: mpsc::Sender>, filter: CatalogMap, pg_xact: Arc>, pg_multixact: Arc>, @@ -479,29 +480,33 @@ async fn gate_task( mut deferred: DeferredSpool, ) -> Result { let mut stats = GateStats::default(); - while let Some(t) = rx.recv().await { - if filter.is_toast(t.rfn.db_node, t.rfn.rel_node) { - if tx.send(t).await.is_err() { - return Ok(stats); + // Slab in, slab out: the gate never breaks a walk slab into per-tuple + // channel hops, it only drops what the visibility verdict removes + while let Some(slab) = rx.recv().await { + let mut pass = Vec::with_capacity(slab.len()); + for t in slab { + if filter.is_toast(t.rfn.db_node, t.rfn.rel_node) { + pass.push(t); + continue; } - continue; - } - match tuple_visibility(t.xid, t.xmax, t.infomask, None) { - Visibility::Emit => { - if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { - stats.multixact_emitted += 1; - } - stats.emitted += 1; - if tx.send(t).await.is_err() { - return Ok(stats); + match tuple_visibility(t.xid, t.xmax, t.infomask, None) { + Visibility::Emit => { + if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { + stats.multixact_emitted += 1; + } + stats.emitted += 1; + pass.push(t); } + Visibility::Skip => stats.gated += 1, + Visibility::Defer => deferred + .push(t) + .await + .map_err(|e| format!("backup_backfill: deferred spool: {e}"))?, + Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), } - Visibility::Skip => stats.gated += 1, - Visibility::Defer => deferred - .push(t) - .await - .map_err(|e| format!("backup_backfill: deferred spool: {e}"))?, - Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), + } + if !pass.is_empty() && tx.send(pass).await.is_err() { + return Ok(stats); } } // Walk EOF: sink dropped. Deferred tuples sit in the spool past its @@ -535,7 +540,7 @@ async fn gate_task( stats.multixact_emitted += 1; } stats.emitted += 1; - if tx.send(t).await.is_err() { + if tx.send(vec![t]).await.is_err() { return Ok(stats); } } @@ -1603,30 +1608,30 @@ mod tests { // Hinted-committed: passes through immediately walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID, - )) + )]) .await .unwrap(); // Hinted-aborted: gated walk_tx - .send(tuple(16400, 101, 0, HEAP_XMIN_INVALID)) + .send(vec![tuple(16400, 101, 0, HEAP_XMIN_INVALID)]) .await .unwrap(); // Unhinted, gap-committed writer: deferred, then emitted via patch - walk_tx.send(tuple(16400, 500, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 500, 0, 0)]).await.unwrap(); // Unhinted, gap-aborted writer: deferred, then gated via patch - walk_tx.send(tuple(16400, 600, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 600, 0, 0)]).await.unwrap(); drop(walk_tx); walk_ok_tx.send(()).unwrap(); let stats = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { - got.push(t.xid); + got.extend(t.iter().map(|x| x.xid)); } assert_eq!(got, [100, 500]); assert_eq!(stats.emitted, 2); @@ -1667,23 +1672,23 @@ mod tests { // Hinted-committed: routed before the failure, stays flushed walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID, - )) + )]) .await .unwrap(); // Unhinted: deferred, must be discarded - walk_tx.send(tuple(16400, 500, 0, 0)).await.unwrap(); + walk_tx.send(vec![tuple(16400, 500, 0, 0)]).await.unwrap(); drop(walk_tx); drop(walk_ok_tx); let stats = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { - got.push(t.xid); + got.extend(t.iter().map(|x| x.xid)); } assert_eq!(got, [100], "deferred tuple not emitted"); assert_eq!(stats.emitted, 1); @@ -1732,12 +1737,12 @@ mod tests { )); walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 10, HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI, - )) + )]) .await .unwrap(); drop(walk_tx); @@ -1772,12 +1777,12 @@ mod tests { )); walk_tx - .send(tuple( + .send(vec![tuple( 16400, 100, 10, HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI, - )) + )]) .await .unwrap(); drop(walk_tx); diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index 8e697228..dda657ca 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -14,6 +14,8 @@ use std::io; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; use async_trait::async_trait; use thiserror::Error; @@ -21,14 +23,14 @@ use tokio::sync::mpsc; use walrus::pg::walparser::{Oid, RelFileNode}; use crate::backfill::backup_source::{ - BackupSink, EndInfo, EntryId, FileAction, FileKind, FileMeta, StartInfo, + BackupSink, EntrySink, FileAction, FileKind, FileMeta, StartInfo, }; use crate::backfill::pg_path::{BaseRelFile, RelFork, parse_base_path}; use crate::decode::heap_decoder::{ ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, decode_block_data, }; use crate::schema::RelDescriptor; -use ahash::{HashMap, HashMapExt}; +use ahash::{HashMap, HashMapExt, HashSet}; /// Heap page size, PG compile-time, identical to wal-rus `BLOCK_SIZE` pub const PAGE_BYTES: usize = 8192; @@ -45,11 +47,16 @@ pub const LP_NORMAL: u8 = 1; /// `pg_toast` regnamespace; TOAST tables ship as `pg_toast_` pub const PG_TOAST_NS: &str = "pg_toast"; -/// Bootstrap tuple channel depth. Small + bounded so a saturated CH -/// inserter parks the page walk (and its source fetch) rather than -/// buffering a whole relation in RAM. Just deep enough to absorb a -/// page's worth of tuples without thrashing wakeups. -pub const BOOTSTRAP_TUPLE_CHANNEL_CAP: usize = 256; +/// Body bytes a segment accumulates before its complete pages walk. Sets +/// the `spawn_blocking` grain and, with the channel depth, the pages in +/// flight; 16 pages decode in well under a millisecond, so the hop cost +/// amortizes without pinning much heap. +pub const SLAB_BYTES: usize = 16 * PAGE_BYTES; +/// Bootstrap tuple channel depth, in slabs. Small + bounded so a saturated +/// CH inserter parks the page walk (and its source fetch) rather than +/// buffering a whole relation in RAM. `SLAB_BYTES * CAP` bounds the heap +/// payload in flight per concurrent segment. +pub const BOOTSTRAP_TUPLE_CHANNEL_CAP: usize = 8; #[derive(Debug, Error)] pub enum PageWalkError { @@ -155,22 +162,62 @@ impl CatalogMap { } } -/// Per-pump counters, operator-visible -#[derive(Debug, Default, Clone)] -pub struct PageWalkStats { - pub files_seen: u64, - pub files_walked: u64, - /// Filenode absent from catalog map, typically a race against the seed - pub files_skipped_unknown_filenode: u64, - pub toast_files_observed: u64, +crate::atomic_stats! { + /// Per-pump counters, operator-visible. Atomic so per-entry walkers + /// share one handle instead of funnelling through the sink lock + pub struct PageWalkStats { + pub files_seen, + pub files_walked, + /// Filenode absent from catalog map, typically a race against the seed + pub files_skipped_unknown_filenode, + /// Filenode outside the mapped set, declined before any page decode + pub files_skipped_unmapped, + pub toast_files_observed, + pub pages_walked, + pub slots_seen, + pub tuples_emitted, + pub tuples_skipped_lp_flag, + pub tuples_skipped_truncated, + /// Trailing partial-page bytes; PG heap files are page-aligned so + /// nonzero is anomalous + pub tail_bytes_dropped, + /// `PageWalker::walk_page` CPU, decode included + pub decode_nanos, + /// Blocked on a free tuple-channel slot: the bootstrap + /// backpressure point, so this is emitter drain time seen by the walk + pub channel_block_nanos, + } +} + +/// Plain tally one `walk_page` fills, published into [`PageWalkStats`] once +/// per slab. Per-tuple `fetch_add` on a line the reader task also touches +/// (the shared `Arc`'s refcount sits in the same allocation) costs more +/// than the decode it counts +#[derive(Debug, Default, Clone, Copy)] +pub struct PageWalkTally { pub pages_walked: u64, pub slots_seen: u64, pub tuples_emitted: u64, pub tuples_skipped_lp_flag: u64, pub tuples_skipped_truncated: u64, - /// Trailing partial-page bytes; PG heap files are page-aligned so - /// nonzero is anomalous - pub tail_bytes_dropped: u64, +} + +impl PageWalkTally { + pub fn publish(&self, stats: &PageWalkStats) { + let add = |c: &AtomicU64, v: u64| { + if v > 0 { + c.fetch_add(v, Ordering::Relaxed); + } + }; + add(&stats.pages_walked, self.pages_walked); + add(&stats.slots_seen, self.slots_seen); + add(&stats.tuples_emitted, self.tuples_emitted); + add(&stats.tuples_skipped_lp_flag, self.tuples_skipped_lp_flag); + add( + &stats.tuples_skipped_truncated, + self.tuples_skipped_truncated, + ); + } } /// Walks one 8 KiB page, emitting `BackfillTuple`s for `LP_NORMAL` @@ -191,7 +238,7 @@ impl<'a> PageWalker<'a> { page: &[u8], block_no: u32, out: &mut Vec, - stats: &mut PageWalkStats, + tally: &mut PageWalkTally, ) -> Result<(), PageWalkError> { if page.len() < SIZE_OF_PAGE_HEADER { return Err(PageWalkError::BadPageHeader { @@ -207,12 +254,12 @@ impl<'a> PageWalker<'a> { let pd_lower = u16::from_le_bytes(page[12..14].try_into().unwrap()); let pd_upper = u16::from_le_bytes(page[14..16].try_into().unwrap()); if pd_upper == 0 && page.iter().all(|&byte| byte == 0) { - stats.pages_walked += 1; + tally.pages_walked += 1; return Ok(()); } if pd_lower as usize == SIZE_OF_PAGE_HEADER && pd_upper as usize == PAGE_BYTES { // Fresh / empty page - stats.pages_walked += 1; + tally.pages_walked += 1; return Ok(()); } if (pd_lower as usize) < SIZE_OF_PAGE_HEADER @@ -226,9 +273,10 @@ impl<'a> PageWalker<'a> { }); } let n_slots = (pd_lower as usize - SIZE_OF_PAGE_HEADER) / SIZE_OF_ITEM_ID; - stats.pages_walked += 1; + tally.pages_walked += 1; + tally.slots_seen += n_slots as u64; + out.reserve(n_slots); for i in 0..n_slots { - stats.slots_seen += 1; let off = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; let raw = u32::from_le_bytes(page[off..off + 4].try_into().unwrap()); // bit-packed: lp_off (15) | lp_flags (2) | lp_len (15) @@ -236,11 +284,11 @@ impl<'a> PageWalker<'a> { let lp_flags = ((raw >> 15) & 0x3) as u8; let lp_len = ((raw >> 17) & 0x7FFF) as usize; if lp_flags != LP_NORMAL { - stats.tuples_skipped_lp_flag += 1; + tally.tuples_skipped_lp_flag += 1; continue; } if lp_off + lp_len > PAGE_BYTES || lp_len == 0 { - stats.tuples_skipped_truncated += 1; + tally.tuples_skipped_truncated += 1; continue; } let tuple_bytes = &page[lp_off..lp_off + lp_len]; @@ -257,9 +305,9 @@ impl<'a> PageWalker<'a> { offnum: (i + 1) as u16, columns, }); - stats.tuples_emitted += 1; + tally.tuples_emitted += 1; } - None => stats.tuples_skipped_truncated += 1, + None => tally.tuples_skipped_truncated += 1, } } Ok(()) @@ -348,29 +396,30 @@ pub(crate) fn decode_on_page_tuple( } } -/// Tap sink that buffers tar entry bytes and walks them 8 KiB at a -/// time, shipping tuples over `out_tx` to an async drain task. +/// Tap router: resolves a segment's descriptor at `begin` and hands back a +/// [`PageWalkEntry`] that owns the walk. Nothing here is on the body path, +/// so concurrent tar parts never queue behind each other. pub struct PageWalkSink { catalog: CatalogMap, - source_lsn: u64, + /// `StartInfo.start_lsn`, set once before any `begin` + source_lsn: AtomicU64, /// Per-rfn `_lsn` tag overriding `source_lsn` (backup-sourced opt-in /// backfills tag each rel with its own boundary; greenfield leaves /// this empty). Keyed `(db_node, rel_node)`. lsn_overrides: HashMap<(Oid, Oid), u64>, - pub stats: PageWalkStats, - /// Bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]): `chunk` is async, so a - /// full channel awaits in `ship_tuple`, parking the source body read + pub stats: Arc, + /// Bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]) slab channel: a full + /// channel awaits in the entry sink, parking that entry's body read /// instead of buffering. Backpressure, not a buffer. - out_tx: Option>, + out_tx: Option>>, /// Test-only capture, populated when `out_tx` is None - pub captured: Vec, - /// Per-entry state keyed by `EntryId`. A map, not one slot, because - /// object_store fan-out interleaves begin/chunk across concurrent - /// parts; one slot would let part B's begin clobber part A's - /// in-flight entry, mis-framing pages and misattributing rfns. - cur: HashMap, + captured: Arc>>, /// Decode TOAST pages only when configured store consumes them store_toast: bool, + /// Filenodes worth decoding, mapped relations plus their TOAST heaps. + /// `None` taps everything the catalog map holds. A superset filter, not a + /// second source of truth: the drain's `skip_initial` stays the authority + tap_filenodes: Option>>, /// `pg_xact/` segments Tap into here for backfill visibility gate /// (plans/add_table.md); `None` (greenfield) keeps Skip. pg_xact: Option>>, @@ -386,36 +435,21 @@ enum SlruSegment { MultiMembers(u32), } -struct TapEntry { - /// Block number within the relation, global across `.N` segments - /// (seeded `segno * RELSEG_BLOCKS`): TOAST rows key on - /// `(blkno, offnum)`, and all walk rows share one LSN, so per-file - /// numbering would collide segment TIDs at equal version - block_no: u32, - /// Carry-over bytes when a chunk read straddled a page boundary - page_buf: Vec, - is_toast: bool, - desc: Option>, - /// `Some` for SLRU entries (`pg_xact/`, `pg_multixact/`): bytes - /// accumulate whole (no page framing) and install at `end()` - slru: Option, -} - impl PageWalkSink { pub fn new( catalog: CatalogMap, - out_tx: mpsc::Sender, + out_tx: mpsc::Sender>, store_toast: bool, ) -> Self { Self { catalog, - source_lsn: 0, + source_lsn: AtomicU64::new(0), lsn_overrides: HashMap::new(), - stats: PageWalkStats::default(), + stats: Arc::new(PageWalkStats::default()), out_tx: Some(out_tx), - captured: Vec::new(), - cur: HashMap::new(), + captured: Arc::default(), store_toast, + tap_filenodes: None, pg_xact: None, pg_multixact: None, } @@ -439,6 +473,20 @@ impl PageWalkSink { self } + /// Decline filenodes outside `set` at `begin`. Bytes still drain off the + /// wire; tuples never decode + pub fn with_tap_filenodes(mut self, set: Arc>) -> Self { + self.tap_filenodes = Some(set); + self + } + + /// Share the counter handle so a caller watches the walk live instead + /// of waiting for the pump to hand its sink back. + pub fn with_stats(mut self, stats: Arc) -> Self { + self.stats = stats; + self + } + /// Tag listed rfns' rows with their own `_lsn` instead of `source_lsn`. pub fn with_lsn_overrides(mut self, overrides: HashMap<(Oid, Oid), u64>) -> Self { self.lsn_overrides = overrides; @@ -450,13 +498,13 @@ impl PageWalkSink { pub fn new_capturing(catalog: CatalogMap) -> Self { Self { catalog, - source_lsn: 0, + source_lsn: AtomicU64::new(0), lsn_overrides: HashMap::new(), - stats: PageWalkStats::default(), + stats: Arc::new(PageWalkStats::default()), out_tx: None, - captured: Vec::new(), - cur: HashMap::new(), + captured: Arc::default(), store_toast: false, + tap_filenodes: None, pg_xact: None, pg_multixact: None, } @@ -471,8 +519,14 @@ impl PageWalkSink { } } + /// Tuples a capturing sink collected. Empty once `out_tx` is wired + #[cfg(test)] + pub fn captured(&self) -> Vec { + self.captured.lock().expect("captured lock").clone() + } + pub fn source_lsn(&self) -> u64 { - self.source_lsn + self.source_lsn.load(Ordering::Relaxed) } fn classify(&self, meta: &FileMeta) -> Option { @@ -497,91 +551,25 @@ impl PageWalkSink { } None } - - async fn flush_full_pages(&mut self, id: EntryId) -> io::Result<()> { - loop { - // Take the page so the entry borrow drops before touching - // self.stats / out_tx.send - let (block_no, is_toast, desc_opt, page) = { - let Some(entry) = self.cur.get_mut(&id) else { - return Ok(()); - }; - if entry.slru.is_some() || entry.page_buf.len() < PAGE_BYTES { - return Ok(()); - } - let block_no = entry.block_no; - entry.block_no = entry.block_no.saturating_add(1); - let page: Vec = entry.page_buf.drain(..PAGE_BYTES).collect(); - (block_no, entry.is_toast, entry.desc.clone(), page) - }; - - if is_toast && !self.store_toast { - self.stats.pages_walked += 1; - continue; - } - let Some(desc) = desc_opt else { - // Filenode absent from seed; stats counted at begin() - continue; - }; - let lsn = self - .lsn_overrides - .get(&(desc.rfn.db_node, desc.rfn.rel_node)) - .copied() - .unwrap_or(self.source_lsn); - let walker = PageWalker::new(&desc, lsn); - let mut local_out = Vec::new(); - if let Err(e) = walker.walk_page(&page, block_no, &mut local_out, &mut self.stats) { - tracing::warn!( - target = "walshadow::backup_page_walk", - block = block_no, - error = %e, - "page walk skipped due to framing error" - ); - continue; - } - for t in local_out { - self.ship_tuple(t).await?; - } - } - } - - async fn ship_tuple(&mut self, t: BackfillTuple) -> io::Result<()> { - match &self.out_tx { - // Awaits a free slot when full: this is the bootstrap - // backpressure point, parking the source until CH drains - Some(tx) => tx.send(t).await.map_err(|e| { - io::Error::other(format!("PageWalkSink: emitter channel closed: {e}")) - }), - None => { - self.captured.push(t); - Ok(()) - } - } - } } #[async_trait] impl BackupSink for PageWalkSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { - self.source_lsn = info.start_lsn; + async fn start(&self, info: &StartInfo) -> io::Result<()> { + self.source_lsn.store(info.start_lsn, Ordering::Relaxed); Ok(()) } - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result { + async fn begin(&self, meta: &FileMeta) -> io::Result { if matches!(meta.kind, FileKind::File) && let Some(slru) = self.classify_slru(&meta.path) { - self.cur.insert( - entry, - TapEntry { - block_no: 0, - page_buf: Vec::with_capacity(meta.size as usize), - is_toast: false, - desc: None, - slru: Some(slru), - }, - ); - return Ok(FileAction::Tap); + return Ok(FileAction::Tap(Box::new(SlruEntry { + seg: slru, + buf: Vec::with_capacity(meta.size as usize), + pg_xact: self.pg_xact.clone(), + pg_multixact: self.pg_multixact.clone(), + }))); } let Some(f) = self.classify(meta) else { // Not base//; multiplex sink falls back to lander @@ -591,86 +579,281 @@ impl BackupSink for PageWalkSink { // fsm/vm carry no tuples; keep them out of the TID-producing walk return Ok(FileAction::Skip); } - self.stats.files_seen += 1; + self.stats.files_seen.fetch_add(1, Ordering::Relaxed); + // Mapping filter first: an unmapped relation would decode every page + // only for the drain to discard it into `unsupported_relations` + if self + .tap_filenodes + .as_ref() + .is_some_and(|set| !set.contains(&(f.db, f.filenode))) + { + self.stats + .files_skipped_unmapped + .fetch_add(1, Ordering::Relaxed); + return Ok(FileAction::Skip); + } let desc = self.catalog.get(f.db, f.filenode); let is_toast = self.catalog.is_toast(f.db, f.filenode); if is_toast { - self.stats.toast_files_observed += 1; + self.stats + .toast_files_observed + .fetch_add(1, Ordering::Relaxed); } else if desc.is_some() { - self.stats.files_walked += 1; + self.stats.files_walked.fetch_add(1, Ordering::Relaxed); } else { // Filenode absent from map: seed race (greenfield) or non-opted // rel (filtered backfill pass, where this is most files). Skip // drains body without page buffering; mux honours the decline - self.stats.files_skipped_unknown_filenode += 1; + self.stats + .files_skipped_unknown_filenode + .fetch_add(1, Ordering::Relaxed); return Ok(FileAction::Skip); } - self.cur.insert( - entry, - TapEntry { - block_no: f.segno.saturating_mul(RELSEG_BLOCKS), - page_buf: Vec::with_capacity(PAGE_BYTES * 2), - is_toast, - desc, - slru: None, + let Some(desc) = desc else { + // TOAST heap whose descriptor the map lacks; count pages, no walk + return Ok(FileAction::Tap(Box::new(PageWalkEntry::counting( + f.segno.saturating_mul(RELSEG_BLOCKS), + self.stats.clone(), + )))); + }; + let lsn = self + .lsn_overrides + .get(&(desc.rfn.db_node, desc.rfn.rel_node)) + .copied() + .unwrap_or_else(|| self.source_lsn()); + Ok(FileAction::Tap(Box::new(PageWalkEntry { + block_no: f.segno.saturating_mul(RELSEG_BLOCKS), + slab: Vec::with_capacity(SLAB_BYTES + PAGE_BYTES), + spare: None, + walk: (!is_toast || self.store_toast).then_some(WalkTarget { desc, lsn }), + out: match &self.out_tx { + Some(tx) => Out::Channel(tx.clone()), + None => Out::Captured(self.captured.clone()), }, - ); - Ok(FileAction::Tap) + stats: self.stats.clone(), + pending: None, + }))) } +} + +/// Descriptor a slab walks against. `None` on a TOAST heap the configured +/// store does not consume: its pages are counted, never decoded +struct WalkTarget { + desc: Arc, + lsn: u64, +} + +enum Out { + Channel(mpsc::Sender>), + Captured(Arc>>), +} + +/// One user-heap segment's walk. Owns its slab, so page framing and decode +/// run without touching the router; the walk itself goes to `spawn_blocking` +/// so tar read and decompress overlap it. +pub struct PageWalkEntry { + /// Block number within the relation, global across `.N` segments + /// (seeded `segno * RELSEG_BLOCKS`): TOAST rows key on + /// `(blkno, offnum)`, and all walk rows share one LSN, so per-file + /// numbering would collide segment TIDs at equal version + block_no: u32, + /// Accumulates body bytes; complete pages walk in place out of it and + /// only a sub-page remainder carries into the next slab + slab: Vec, + /// The slab last handed to the walk, back for reuse. Two buffers + /// ping-pong, so a segment allocates twice however long it is + spare: Option>, + walk: Option, + out: Out, + stats: Arc, + /// Previous slab's walk, still running. One slab of overlap: the tar + /// reader refills while the blocking pool decodes, so read and decode + /// cost `max`, not `sum` + pending: Option, Vec)>>, +} - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()> { - match self.cur.get_mut(&entry) { - Some(e) => e.page_buf.extend_from_slice(bytes), - None => return Err(io::Error::other("PageWalkSink: chunk before begin")), +impl PageWalkEntry { + /// Counts pages without decoding them + fn counting(block_no: u32, stats: Arc) -> Self { + Self { + block_no, + slab: Vec::with_capacity(SLAB_BYTES + PAGE_BYTES), + spare: None, + walk: None, + out: Out::Captured(Arc::default()), + stats, + pending: None, } - self.flush_full_pages(entry).await?; - Ok(()) } - async fn end(&mut self, entry: EntryId) -> io::Result<()> { - if let Some(e) = self.cur.remove(&entry) { - match e.slru { - Some(SlruSegment::PgXact(segno)) => { - if let Some(accum) = &self.pg_xact { - accum - .lock() - .expect("pg_xact accum lock") - .insert_segment(segno, e.page_buf); - } - return Ok(()); - } - Some(SlruSegment::MultiOffsets(segno)) => { - if let Some(accum) = &self.pg_multixact { - accum - .lock() - .expect("pg_multixact accum lock") - .insert_offsets_segment(segno, e.page_buf); - } - return Ok(()); - } - Some(SlruSegment::MultiMembers(segno)) => { - if let Some(accum) = &self.pg_multixact { - accum - .lock() - .expect("pg_multixact accum lock") - .insert_members_segment(segno, e.page_buf); - } - return Ok(()); + /// Collect the in-flight walk: recover its slab and ship its tuples. + /// Shipping here is what carries emitter backpressure back to the read + async fn join_pending(&mut self) -> io::Result<()> { + let Some(handle) = self.pending.take() else { + return Ok(()); + }; + let (walked, tuples) = handle.await.map_err(io::Error::other)?; + self.spare = Some(walked); + self.ship(tuples).await + } + + /// Hand every complete page in the slab to the walk, carrying the + /// sub-page remainder into the next slab. + async fn drain_slab(&mut self) -> io::Result<()> { + let full = self.slab.len() / PAGE_BYTES; + if full == 0 { + return Ok(()); + } + let take = full * PAGE_BYTES; + let first_block = self.block_no; + self.block_no = self.block_no.saturating_add(full as u32); + + let Some(target) = self.walk.as_ref() else { + self.stats + .pages_walked + .fetch_add(full as u64, Ordering::Relaxed); + self.slab.drain(..take); + return Ok(()); + }; + let desc = target.desc.clone(); + let lsn = target.lsn; + + // At most one walk in flight, so a segment holds two slabs, never more + self.join_pending().await?; + let mut next = self + .spare + .take() + .unwrap_or_else(|| Vec::with_capacity(SLAB_BYTES + PAGE_BYTES)); + next.clear(); + next.extend_from_slice(&self.slab[take..]); + let mut walked = std::mem::replace(&mut self.slab, next); + walked.truncate(take); + + let stats = self.stats.clone(); + // Pure CPU over a byte slice; off the async runtime so the drain + // stage it feeds keeps a worker to itself + self.pending = Some(tokio::task::spawn_blocking(move || { + let walker = PageWalker::new(&desc, lsn); + let mut out = Vec::new(); + let mut tally = PageWalkTally::default(); + let started = Instant::now(); + for i in 0..full { + let page = &walked[i * PAGE_BYTES..(i + 1) * PAGE_BYTES]; + let block = first_block.saturating_add(i as u32); + if let Err(e) = walker.walk_page(page, block, &mut out, &mut tally) { + tracing::warn!( + target = "walshadow::backup_page_walk", + block, + error = %e, + "page walk skipped due to framing error" + ); } - None => {} } - let trailing = e.page_buf.len() as u64; - if trailing > 0 { - // PG heap files are page-aligned; trailing bytes are - // zero-padding or anomalous, so count without decoding - self.stats.tail_bytes_dropped += trailing; + tally.publish(&stats); + stats + .decode_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + (walked, out) + })); + Ok(()) + } + + async fn ship(&self, tuples: Vec) -> io::Result<()> { + if tuples.is_empty() { + return Ok(()); + } + match &self.out { + // Awaits a free slot when full: this is the bootstrap + // backpressure point, parking this segment until CH drains + Out::Channel(tx) => { + let started = Instant::now(); + let res = tx.send(tuples).await.map_err(|e| { + io::Error::other(format!("PageWalkSink: emitter channel closed: {e}")) + }); + self.stats + .channel_block_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + res + } + Out::Captured(buf) => { + buf.lock().expect("captured lock").extend(tuples); + Ok(()) } } + } +} + +#[async_trait] +impl EntrySink for PageWalkEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.slab.extend_from_slice(bytes); + if self.slab.len() >= SLAB_BYTES { + self.drain_slab().await?; + } + Ok(()) + } + + async fn end(mut self: Box) -> io::Result<()> { + self.drain_slab().await?; + self.join_pending().await?; + let trailing = self.slab.len() as u64; + if trailing > 0 { + // PG heap files are page-aligned; trailing bytes are + // zero-padding or anomalous, so count without decoding + self.stats + .tail_bytes_dropped + .fetch_add(trailing, Ordering::Relaxed); + } + Ok(()) + } +} + +/// `pg_xact` / `pg_multixact` segment: no page framing, bytes accumulate +/// whole and install into the visibility accum at `end` +struct SlruEntry { + seg: SlruSegment, + buf: Vec, + pg_xact: Option>>, + pg_multixact: Option>>, +} + +#[async_trait] +impl EntrySink for SlruEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.buf.extend_from_slice(bytes); Ok(()) } - async fn finish(&mut self, _info: &EndInfo) -> io::Result<()> { - // Channel close happens when caller drops this sink's Arc + async fn end(self: Box) -> io::Result<()> { + let SlruEntry { + seg, + buf, + pg_xact, + pg_multixact, + } = *self; + match seg { + SlruSegment::PgXact(segno) => { + if let Some(a) = pg_xact { + a.lock() + .expect("pg_xact accum lock") + .insert_segment(segno, buf); + } + } + SlruSegment::MultiOffsets(segno) => { + if let Some(a) = pg_multixact { + a.lock() + .expect("pg_multixact accum lock") + .insert_offsets_segment(segno, buf); + } + } + SlruSegment::MultiMembers(segno) => { + if let Some(a) = pg_multixact { + a.lock() + .expect("pg_multixact accum lock") + .insert_members_segment(segno, buf); + } + } + } Ok(()) } } @@ -737,12 +920,35 @@ pub(crate) fn synth_single_tuple_page(value: i32) -> [u8; PAGE_BYTES] { #[cfg(test)] mod tests { use super::*; + + use crate::backfill::backup_source::EndInfo; + + fn ld(a: &AtomicU64) -> u64 { + a.load(Ordering::Relaxed) + } + + /// `begin` must tap; hands back the owned entry sink + async fn tap(sink: &PageWalkSink, meta: &FileMeta) -> Box { + match sink.begin(meta).await.unwrap() { + FileAction::Tap(e) => e, + other => panic!("expected Tap for {}, got {other:?}", meta.path.display()), + } + } + + fn heap_meta(path: &str) -> FileMeta { + FileMeta { + path: PathBuf::from(path), + size: PAGE_BYTES as u64, + mode: 0o600, + kind: FileKind::File, + } + } use crate::schema::RelName; use std::path::PathBuf; #[tokio::test] async fn source_lsn_reflects_start_info() { - let mut sink = PageWalkSink::new_capturing(CatalogMap::new()); + let sink = PageWalkSink::new_capturing(CatalogMap::new()); assert_eq!(sink.source_lsn(), 0); sink.start(&StartInfo { start_lsn: 0xABCD_1234, @@ -760,15 +966,15 @@ mod tests { let walker = PageWalker::new(&rel, 0xABCD); let page = synth_single_tuple_page(42); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert_eq!(out.len(), 1); assert_eq!(out[0].source_lsn, 0xABCD); assert_eq!(out[0].xid, 99); assert_eq!(out[0].columns.len(), 1); assert!(matches!(out[0].columns[0], Some(ColumnValue::Int4(42)))); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.tuples_emitted, 1); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.tuples_emitted, 1); } #[test] @@ -780,11 +986,11 @@ mod tests { page[12..14].copy_from_slice(&(SIZE_OF_PAGE_HEADER as u16).to_le_bytes()); page[14..16].copy_from_slice(&(PAGE_BYTES as u16).to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert!(out.is_empty()); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.slots_seen, 0); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.slots_seen, 0); } #[test] @@ -792,13 +998,13 @@ mod tests { let rel = make_rel(); let walker = PageWalker::new(&rel, 0); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); + let mut tally = PageWalkTally::default(); walker - .walk_page(&[0; PAGE_BYTES], 0, &mut out, &mut stats) + .walk_page(&[0; PAGE_BYTES], 0, &mut out, &mut tally) .unwrap(); assert!(out.is_empty()); - assert_eq!(stats.pages_walked, 1); - assert_eq!(stats.slots_seen, 0); + assert_eq!(tally.pages_walked, 1); + assert_eq!(tally.slots_seen, 0); } #[test] @@ -817,11 +1023,11 @@ mod tests { let new_raw = lp_off | (3u32 << 15) | (lp_len << 17); page[SIZE_OF_PAGE_HEADER..SIZE_OF_PAGE_HEADER + 4].copy_from_slice(&new_raw.to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - walker.walk_page(&page, 0, &mut out, &mut stats).unwrap(); + let mut tally = PageWalkTally::default(); + walker.walk_page(&page, 0, &mut out, &mut tally).unwrap(); assert!(out.is_empty()); - assert_eq!(stats.tuples_skipped_lp_flag, 1); - assert_eq!(stats.tuples_emitted, 0); + assert_eq!(tally.tuples_skipped_lp_flag, 1); + assert_eq!(tally.tuples_emitted, 0); } #[test] @@ -833,8 +1039,8 @@ mod tests { page[12..14].copy_from_slice(&(PAGE_BYTES as u16).to_le_bytes()); page[14..16].copy_from_slice(&(SIZE_OF_PAGE_HEADER as u16).to_le_bytes()); let mut out = Vec::new(); - let mut stats = PageWalkStats::default(); - let err = walker.walk_page(&page, 0, &mut out, &mut stats); + let mut tally = PageWalkTally::default(); + let err = walker.walk_page(&page, 0, &mut out, &mut tally); assert!(matches!(err, Err(PageWalkError::BadPageHeader { .. }))); } @@ -860,7 +1066,7 @@ mod tests { async fn pagewalk_sink_decodes_one_page_via_chunk_stream() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1234_5678, timeline: 1, @@ -868,20 +1074,12 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(0); - let action = sink.begin(id, &meta).await.unwrap(); - assert_eq!(action, FileAction::Tap); + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; let page = synth_single_tuple_page(99); // Two chunks exercise the buffer-across-chunk path - sink.chunk(id, &page[..4096]).await.unwrap(); - sink.chunk(id, &page[4096..]).await.unwrap(); - sink.end(id).await.unwrap(); + entry.chunk(&page[..4096]).await.unwrap(); + entry.chunk(&page[4096..]).await.unwrap(); + entry.end().await.unwrap(); sink.finish(&EndInfo { end_lsn: 0, timeline: 1, @@ -889,17 +1087,18 @@ mod tests { .await .unwrap(); - assert_eq!(sink.captured.len(), 1); - assert_eq!(sink.captured[0].source_lsn, 0x1234_5678); - assert_eq!(sink.captured[0].xid, 99); + let captured = sink.captured(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].source_lsn, 0x1234_5678); + assert_eq!(captured[0].xid, 99); assert!(matches!( - sink.captured[0].columns[0], + captured[0].columns[0], Some(ColumnValue::Int4(99)) )); - assert_eq!(sink.stats.files_seen, 1); - assert_eq!(sink.stats.files_walked, 1); - assert_eq!(sink.stats.pages_walked, 1); - assert_eq!(sink.stats.tuples_emitted, 1); + assert_eq!(ld(&sink.stats.files_seen), 1); + assert_eq!(ld(&sink.stats.files_walked), 1); + assert_eq!(ld(&sink.stats.pages_walked), 1); + assert_eq!(ld(&sink.stats.tuples_emitted), 1); } /// `.N` segment tuples get global block numbers: same local page + @@ -909,7 +1108,7 @@ mod tests { async fn pagewalk_sink_seeds_segment_block_numbers() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -917,26 +1116,20 @@ mod tests { }) .await .unwrap(); - for (i, path) in ["base/5/16400", "base/5/16400.1"].iter().enumerate() { - let meta = FileMeta { - path: PathBuf::from(path), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(i as u64); - assert_eq!(sink.begin(id, &meta).await.unwrap(), FileAction::Tap); - sink.chunk(id, &synth_single_tuple_page(7)).await.unwrap(); - sink.end(id).await.unwrap(); + for path in ["base/5/16400", "base/5/16400.1"] { + let mut entry = tap(&sink, &heap_meta(path)).await; + entry.chunk(&synth_single_tuple_page(7)).await.unwrap(); + entry.end().await.unwrap(); } - assert_eq!(sink.captured.len(), 2); + let captured = sink.captured(); + assert_eq!(captured.len(), 2); assert_eq!( - (sink.captured[0].blkno, sink.captured[0].offnum), + (captured[0].blkno, captured[0].offnum), (0, 1), "base file walks from block 0" ); assert_eq!( - (sink.captured[1].blkno, sink.captured[1].offnum), + (captured[1].blkno, captured[1].offnum), (RELSEG_BLOCKS, 1), "segment 1 walks from its global block" ); @@ -947,7 +1140,7 @@ mod tests { async fn pagewalk_sink_skips_non_main_forks() { let mut catalog = CatalogMap::new(); catalog.insert(Arc::new(make_rel())); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -956,19 +1149,15 @@ mod tests { .await .unwrap(); for path in ["base/5/16400_fsm", "base/5/16400_vm", "base/5/16400_vm.1"] { - let meta = FileMeta { - path: PathBuf::from(path), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - assert_eq!( - sink.begin(EntryId(9), &meta).await.unwrap(), - FileAction::Skip, + assert!( + matches!( + sink.begin(&heap_meta(path)).await.unwrap(), + FileAction::Skip + ), "{path}" ); } - assert_eq!(sink.stats.files_seen, 0); + assert_eq!(ld(&sink.stats.files_seen), 0); } #[tokio::test] @@ -990,7 +1179,6 @@ mod tests { }) ); - let mut sink = sink; sink.start(&StartInfo { start_lsn: 0, timeline: 1, @@ -998,28 +1186,97 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let id = EntryId(0); - let action = sink.begin(id, &meta).await.unwrap(); // Skip so source drains body without chunk() delivery; filtered // backfill pass relies on this for every non-opted rel - assert_eq!(action, FileAction::Skip); - sink.end(id).await.unwrap(); - assert!(sink.captured.is_empty()); - assert_eq!(sink.stats.files_seen, 1); - assert_eq!(sink.stats.files_skipped_unknown_filenode, 1); - assert_eq!(sink.stats.tuples_emitted, 0); - assert_eq!(sink.stats.pages_walked, 0); + assert!(matches!( + sink.begin(&heap_meta("base/5/16400")).await.unwrap(), + FileAction::Skip + )); + assert!(sink.captured().is_empty()); + assert_eq!(ld(&sink.stats.files_seen), 1); + assert_eq!(ld(&sink.stats.files_skipped_unknown_filenode), 1); + assert_eq!(ld(&sink.stats.tuples_emitted), 0); + assert_eq!(ld(&sink.stats.pages_walked), 0); + } + + /// Item 3.4: an unmapped relation's bytes drain off the wire without a + /// page decode. Before the filter the walk decoded every seeded rel and + /// the drain discarded them into `unsupported_relations`. + #[tokio::test] + async fn tap_filenode_filter_declines_unmapped_relations() { + let mut catalog = CatalogMap::new(); + let mapped = make_rel(); + let mut unmapped = make_rel(); + unmapped.rfn.rel_node = 16401; + unmapped.oid = 16401; + catalog.insert(Arc::new(mapped)); + catalog.insert(Arc::new(unmapped)); + let sink = PageWalkSink::new_capturing(catalog) + .with_tap_filenodes(Arc::new([(5, 16400)].into_iter().collect())); + sink.start(&StartInfo { + start_lsn: 0x1000, + timeline: 1, + tablespaces: Vec::new(), + }) + .await + .unwrap(); + + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; + entry.chunk(&synth_single_tuple_page(1)).await.unwrap(); + entry.end().await.unwrap(); + assert!(matches!( + sink.begin(&heap_meta("base/5/16401")).await.unwrap(), + FileAction::Skip + )); + + assert_eq!(sink.captured().len(), 1, "only the mapped rel decoded"); + assert_eq!(ld(&sink.stats.files_seen), 2); + assert_eq!(ld(&sink.stats.files_walked), 1); + assert_eq!(ld(&sink.stats.files_skipped_unmapped), 1); + assert_eq!(ld(&sink.stats.pages_walked), 1); + } + + /// A slab boundary must not lose the page it straddles, nor renumber + /// blocks: 64 KiB chunks against a 16-page slab put the split mid-page. + #[tokio::test] + async fn slab_boundary_keeps_every_page_and_block_number() { + let mut catalog = CatalogMap::new(); + catalog.insert(Arc::new(make_rel())); + let sink = PageWalkSink::new_capturing(catalog); + sink.start(&StartInfo { + start_lsn: 0x1000, + timeline: 1, + tablespaces: Vec::new(), + }) + .await + .unwrap(); + + // 40 pages, one tuple each, delivered in 6000-byte chunks so no + // chunk edge lands on a page or slab edge + let pages = 40usize; + let mut body = Vec::with_capacity(pages * PAGE_BYTES); + for i in 0..pages { + body.extend_from_slice(&synth_single_tuple_page(i as i32)); + } + let mut entry = tap(&sink, &heap_meta("base/5/16400")).await; + for chunk in body.chunks(6000) { + entry.chunk(chunk).await.unwrap(); + } + entry.end().await.unwrap(); + + let captured = sink.captured(); + assert_eq!(captured.len(), pages); + assert_eq!(ld(&sink.stats.pages_walked), pages as u64); + assert_eq!(ld(&sink.stats.tail_bytes_dropped), 0); + for (i, t) in captured.iter().enumerate() { + assert_eq!(t.blkno, i as u32, "page {i} kept its block number"); + assert!(matches!(t.columns[0], Some(ColumnValue::Int4(v)) if v == i as i32)); + } } #[tokio::test] async fn pagewalk_sink_rejects_non_base_paths() { - let mut sink = PageWalkSink::new_capturing(CatalogMap::new()); + let sink = PageWalkSink::new_capturing(CatalogMap::new()); sink.start(&StartInfo { start_lsn: 0, timeline: 1, @@ -1027,23 +1284,23 @@ mod tests { }) .await .unwrap(); - let meta = FileMeta { - path: PathBuf::from("pg_control"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!( - sink.begin(EntryId(0), &meta).await.unwrap(), + assert!(matches!( + sink.begin(&FileMeta { + path: PathBuf::from("pg_control"), + size: 0, + mode: 0, + kind: FileKind::File, + }) + .await + .unwrap(), FileAction::Skip - ); + )); } /// object_store fan-out interleaves begin/chunk across concurrent - /// parts on the shared sink (mutex released across each body read). - /// Per-entry keying keeps each file's page-walk state separate; a - /// single `cur` slot would let a later begin clobber an in-flight - /// entry, misframing pages against the wrong relation. + /// parts. Each entry owns its walk state, so a later begin cannot + /// clobber an in-flight entry and misframe pages against the wrong + /// relation. #[tokio::test] async fn interleaved_entries_keep_independent_state() { let mut catalog = CatalogMap::new(); @@ -1053,7 +1310,7 @@ mod tests { rel_b.oid = 16401; catalog.insert(Arc::new(rel_a)); catalog.insert(Arc::new(rel_b)); - let mut sink = PageWalkSink::new_capturing(catalog); + let sink = PageWalkSink::new_capturing(catalog); sink.start(&StartInfo { start_lsn: 0x1000, timeline: 1, @@ -1062,38 +1319,25 @@ mod tests { .await .unwrap(); - let meta_a = FileMeta { - path: PathBuf::from("base/5/16400"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let meta_b = FileMeta { - path: PathBuf::from("base/5/16401"), - size: PAGE_BYTES as u64, - mode: 0o600, - kind: FileKind::File, - }; - let (a, b) = (EntryId(0), EntryId(1)); - // Both open before either streams, chunks arrive reversed, ends // interleave: worst case for a shared slot - assert_eq!(sink.begin(a, &meta_a).await.unwrap(), FileAction::Tap); - assert_eq!(sink.begin(b, &meta_b).await.unwrap(), FileAction::Tap); - sink.chunk(b, &synth_single_tuple_page(200)).await.unwrap(); - sink.chunk(a, &synth_single_tuple_page(100)).await.unwrap(); - sink.end(a).await.unwrap(); - sink.end(b).await.unwrap(); + let mut a = tap(&sink, &heap_meta("base/5/16400")).await; + let mut b = tap(&sink, &heap_meta("base/5/16401")).await; + b.chunk(&synth_single_tuple_page(200)).await.unwrap(); + a.chunk(&synth_single_tuple_page(100)).await.unwrap(); + a.end().await.unwrap(); + b.end().await.unwrap(); // Each tuple carries its own file's rfn + value; a shared slot // would attribute both to rel B + let captured = sink.captured(); let mut by_rel: HashMap = HashMap::new(); - for t in &sink.captured { + for t in &captured { if let Some(Some(ColumnValue::Int4(v))) = t.columns.first() { by_rel.insert(t.rfn.rel_node, *v); } } - assert_eq!(sink.captured.len(), 2); + assert_eq!(captured.len(), 2); assert_eq!(by_rel.get(&16400), Some(&100), "entry A decoded vs rel A"); assert_eq!(by_rel.get(&16401), Some(&200), "entry B decoded vs rel B"); } diff --git a/src/backfill/backup_sink.rs b/src/backfill/backup_sink.rs index d1759200..6caa9f9a 100644 --- a/src/backfill/backup_sink.rs +++ b/src/backfill/backup_sink.rs @@ -11,15 +11,17 @@ //! `CatalogTracker::seed_from_source`. use std::io; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use async_trait::async_trait; use crate::backfill::backup_source::{ - BackupSink, EndInfo, EntryId, FileAction, FileKind, FileMeta, StartInfo, + BackupSink, EndInfo, FileAction, FileKind, FileMeta, StartInfo, }; use crate::backfill::pg_path::{is_system_dir, parse_base_path}; use crate::schema::FIRST_NORMAL_OBJECT_ID; -use ahash::{HashSet, HashSetExt}; +use ahash::HashSet; /// `relfilenode < 16384` is always bootstrap catalog. Rotated-catalog /// filenodes (`VACUUM FULL` / `REINDEX` on a catalog) land in `whitelist`. @@ -71,23 +73,24 @@ impl FromIterator<(u32, u32)> for CatalogFilenodes { /// (PG recovery refuses to start without them). pub struct DiskLanderSink { pub catalog_filenodes: CatalogFilenodes, - pub stats: DiskLanderStats, + pub stats: Arc, } -#[derive(Debug, Default, Clone)] -pub struct DiskLanderStats { - pub kept_files: u64, - pub kept_dirs: u64, - pub kept_symlinks: u64, - pub skipped_denylist: u64, - pub skipped_user_heap: u64, +crate::atomic_stats! { + pub struct DiskLanderStats { + pub kept_files, + pub kept_dirs, + pub kept_symlinks, + pub skipped_denylist, + pub skipped_user_heap, + } } impl DiskLanderSink { pub fn new(catalog_filenodes: CatalogFilenodes) -> Self { Self { catalog_filenodes, - stats: DiskLanderStats::default(), + stats: Arc::new(DiskLanderStats::default()), } } @@ -117,7 +120,7 @@ impl DiskLanderSink { } /// Distinguishes skip-denylist from skip-user-heap so MultiplexSink can -/// flip the latter to `Tap` when a page-walk sink is composed in. +/// flip the latter to a tap when a page-walk sink is composed in. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DiskAction { Keep, @@ -125,64 +128,56 @@ pub enum DiskAction { SkipUserHeap, } +impl DiskLanderSink { + /// Count one routing decision. Split out so `MultiplexSink` records the + /// lander's view of an entry it hands to the tap instead + fn record(&self, action: DiskAction, meta: &FileMeta) { + let bump = |c: &AtomicU64| { + c.fetch_add(1, Ordering::Relaxed); + }; + match action { + DiskAction::Keep => match meta.kind { + FileKind::File => bump(&self.stats.kept_files), + FileKind::Dir => bump(&self.stats.kept_dirs), + FileKind::Symlink { .. } => bump(&self.stats.kept_symlinks), + }, + DiskAction::SkipDenylist => bump(&self.stats.skipped_denylist), + DiskAction::SkipUserHeap => bump(&self.stats.skipped_user_heap), + } + } +} + #[async_trait] impl BackupSink for DiskLanderSink { - async fn begin(&mut self, _entry: EntryId, meta: &FileMeta) -> io::Result { - let action = match self.classify(meta) { + async fn begin(&self, meta: &FileMeta) -> io::Result { + let action = self.classify(meta); + self.record(action, meta); + Ok(match action { DiskAction::Keep => FileAction::Keep, DiskAction::SkipDenylist | DiskAction::SkipUserHeap => FileAction::Skip, - }; - if action == FileAction::Keep { - match meta.kind { - FileKind::File => self.stats.kept_files += 1, - FileKind::Dir => self.stats.kept_dirs += 1, - FileKind::Symlink { .. } => self.stats.kept_symlinks += 1, - } - } else { - match self.classify(meta) { - DiskAction::SkipDenylist => self.stats.skipped_denylist += 1, - DiskAction::SkipUserHeap => self.stats.skipped_user_heap += 1, - DiskAction::Keep => unreachable!(), - } - } - Ok(action) - } - async fn chunk(&mut self, _entry: EntryId, _bytes: &[u8]) -> io::Result<()> { - // Never returns Tap, so chunk() should never fire - Err(io::Error::other( - "DiskLanderSink::chunk called — sink only ever Keeps or Skips", - )) - } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - Ok(()) + }) } } -/// Multiplexes a DiskLanderSink (Keep / Skip) and a Tap-target sink -/// (typically PageWalkSink) over one source pass; begin() picks the -/// route, chunk/end follow it. +/// Routes a DiskLanderSink (Keep / Skip) and a tap-target sink +/// (typically PageWalkSink) over one source pass. Pure router: the tap's +/// own `EntrySink` carries the body, so nothing here is on the hot path. pub struct MultiplexSink { lander: DiskLanderSink, tap: T, - /// Entries routed to the tap. A set, not a flag, so concurrent - /// entries (object_store fan-out) dispatch to the right inner sink - /// instead of racing one shared bool. - tap_entries: HashSet, } impl MultiplexSink { pub fn new(lander: DiskLanderSink, tap: T) -> Self { - Self { - lander, - tap, - tap_entries: HashSet::new(), - } + Self { lander, tap } } - pub fn lander_stats(&self) -> &DiskLanderStats { + #[cfg(test)] + pub fn lander_stats(&self) -> &Arc { &self.lander.stats } + #[cfg(test)] pub fn into_inner(self) -> (DiskLanderSink, T) { (self.lander, self.tap) } @@ -190,49 +185,25 @@ impl MultiplexSink { #[async_trait] impl BackupSink for MultiplexSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { + async fn start(&self, info: &StartInfo) -> io::Result<()> { self.lander.start(info).await?; self.tap.start(info).await?; Ok(()) } - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result { - let action = match self.lander.classify(meta) { - DiskAction::Keep => { - self.lander.begin(entry, meta).await?; - FileAction::Keep - } - DiskAction::SkipDenylist => { - self.lander.begin(entry, meta).await?; - FileAction::Skip - } - DiskAction::SkipUserHeap => { - // Flip to Tap if the inner sink accepts; honour its - // decline (Skip / Keep) otherwise - let inner_action = self.tap.begin(entry, meta).await?; - if inner_action == FileAction::Tap { - self.tap_entries.insert(entry); - } - inner_action - } - }; - Ok(action) - } - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()> { - if self.tap_entries.contains(&entry) { - self.tap.chunk(entry, bytes).await - } else { - Err(io::Error::other("MultiplexSink: chunk without active tap")) + async fn begin(&self, meta: &FileMeta) -> io::Result { + let disk = self.lander.classify(meta); + if disk != DiskAction::SkipUserHeap { + return self.lander.begin(meta).await; } - } - async fn end(&mut self, entry: EntryId) -> io::Result<()> { - if self.tap_entries.remove(&entry) { - self.tap.end(entry).await?; - } else { - self.lander.end(entry).await?; + // Hand it to the tap; honour its decline (Skip / Keep) otherwise, + // counting the lander's own verdict only when it keeps the entry + let inner = self.tap.begin(meta).await?; + if !inner.is_tap() { + self.lander.record(disk, meta); } - Ok(()) + Ok(inner) } - async fn finish(&mut self, info: &EndInfo) -> io::Result<()> { + async fn finish(&self, info: &EndInfo) -> io::Result<()> { self.lander.finish(info).await?; self.tap.finish(info).await?; Ok(()) @@ -414,28 +385,43 @@ mod tests { } } - /// Counts begin / chunk / end, always returns Tap. Exercises - /// MultiplexSink without the full page walker. + fn ld(a: &AtomicU64) -> u64 { + a.load(Ordering::Relaxed) + } + + /// Counts begin / chunk / end, always taps. Exercises MultiplexSink + /// without the full page walker. #[derive(Debug, Default)] - struct CountingTap { - begins: u64, - chunks: u64, - ends: u64, - bytes: u64, + struct TapCounts { + begins: AtomicU64, + chunks: AtomicU64, + ends: AtomicU64, + bytes: AtomicU64, } + + struct CountingTap(Arc); + #[async_trait] impl BackupSink for CountingTap { - async fn begin(&mut self, _entry: EntryId, _meta: &FileMeta) -> io::Result { - self.begins += 1; - Ok(FileAction::Tap) + async fn begin(&self, _meta: &FileMeta) -> io::Result { + self.0.begins.fetch_add(1, Ordering::Relaxed); + Ok(FileAction::Tap(Box::new(CountingEntry(self.0.clone())))) } - async fn chunk(&mut self, _entry: EntryId, bytes: &[u8]) -> io::Result<()> { - self.chunks += 1; - self.bytes += bytes.len() as u64; + } + + struct CountingEntry(Arc); + + #[async_trait] + impl crate::backfill::backup_source::EntrySink for CountingEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.0.chunks.fetch_add(1, Ordering::Relaxed); + self.0 + .bytes + .fetch_add(bytes.len() as u64, Ordering::Relaxed); Ok(()) } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - self.ends += 1; + async fn end(self: Box) -> io::Result<()> { + self.0.ends.fetch_add(1, Ordering::Relaxed); Ok(()) } } @@ -443,49 +429,43 @@ mod tests { #[tokio::test] async fn multiplex_sink_routes_user_heap_to_tap() { let lander = DiskLanderSink::new(CatalogFilenodes::new()); - let tap = CountingTap::default(); - let mut mux = MultiplexSink::new(lander, tap); - - let m = FileMeta { - path: PathBuf::from("base/5/1259"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!(mux.begin(EntryId(0), &m).await.unwrap(), FileAction::Keep); - mux.end(EntryId(0)).await.unwrap(); + let counts = Arc::new(TapCounts::default()); + let mux = MultiplexSink::new(lander, CountingTap(counts.clone())); - let m = FileMeta { - path: PathBuf::from("base/5/16400"), + let file = |p: &str| FileMeta { + path: PathBuf::from(p), size: 0, mode: 0, kind: FileKind::File, }; - assert_eq!(mux.begin(EntryId(1), &m).await.unwrap(), FileAction::Tap); - mux.chunk(EntryId(1), &[0u8; 1024]).await.unwrap(); - mux.chunk(EntryId(1), &[1u8; 512]).await.unwrap(); - mux.end(EntryId(1)).await.unwrap(); + assert!(matches!( + mux.begin(&file("base/5/1259")).await.unwrap(), + FileAction::Keep + )); - let m = FileMeta { - path: PathBuf::from("pg_replslot/0/state"), - size: 0, - mode: 0, - kind: FileKind::File, + let FileAction::Tap(mut entry) = mux.begin(&file("base/5/16400")).await.unwrap() else { + panic!("user heap must tap"); }; - assert_eq!(mux.begin(EntryId(2), &m).await.unwrap(), FileAction::Skip); - mux.end(EntryId(2)).await.unwrap(); - - let (lander, tap) = mux.into_inner(); - assert_eq!(tap.begins, 1); - assert_eq!(tap.chunks, 2); - assert_eq!(tap.ends, 1); - assert_eq!(tap.bytes, 1536); - assert_eq!(lander.stats.kept_files, 1); - // User heap delegated to tap; lander never begin'd it so - // skipped_user_heap stays 0. tap.begins == 1 is the - // operator-visible "routed away from disk" signal. - assert_eq!(lander.stats.skipped_user_heap, 0); - assert_eq!(lander.stats.skipped_denylist, 1); + entry.chunk(&[0u8; 1024]).await.unwrap(); + entry.chunk(&[1u8; 512]).await.unwrap(); + entry.end().await.unwrap(); + + assert!(matches!( + mux.begin(&file("pg_replslot/0/state")).await.unwrap(), + FileAction::Skip + )); + + let (lander, _) = mux.into_inner(); + assert_eq!(ld(&counts.begins), 1); + assert_eq!(ld(&counts.chunks), 2); + assert_eq!(ld(&counts.ends), 1); + assert_eq!(ld(&counts.bytes), 1536); + assert_eq!(ld(&lander.stats.kept_files), 1); + // User heap delegated to tap, so the lander's own skip counter + // stays 0. `counts.begins == 1` is the operator-visible + // "routed away from disk" signal. + assert_eq!(ld(&lander.stats.skipped_user_heap), 0); + assert_eq!(ld(&lander.stats.skipped_denylist), 1); } #[test] @@ -505,26 +485,18 @@ mod tests { #[tokio::test] async fn multiplex_lander_stats_exposes_disk_counters() { let lander = DiskLanderSink::new(CatalogFilenodes::new()); - let mut mux = MultiplexSink::new(lander, CountingTap::default()); - let m = FileMeta { - path: PathBuf::from("base/5/1259"), - size: 0, - mode: 0, - kind: FileKind::File, - }; - assert_eq!(mux.begin(EntryId(0), &m).await.unwrap(), FileAction::Keep); - mux.end(EntryId(0)).await.unwrap(); - let m = FileMeta { - path: PathBuf::from("pg_replslot/0/state"), + let mux = MultiplexSink::new(lander, CountingTap(Arc::default())); + let file = |p: &str| FileMeta { + path: PathBuf::from(p), size: 0, mode: 0, kind: FileKind::File, }; - assert_eq!(mux.begin(EntryId(1), &m).await.unwrap(), FileAction::Skip); - mux.end(EntryId(1)).await.unwrap(); + mux.begin(&file("base/5/1259")).await.unwrap(); + mux.begin(&file("pg_replslot/0/state")).await.unwrap(); let stats = mux.lander_stats(); - assert_eq!(stats.kept_files, 1); - assert_eq!(stats.skipped_denylist, 1); + assert_eq!(ld(&stats.kept_files), 1); + assert_eq!(ld(&stats.skipped_denylist), 1); } } diff --git a/src/backfill/backup_source.rs b/src/backfill/backup_source.rs index 7fc98f2d..1470a1df 100644 --- a/src/backfill/backup_source.rs +++ b/src/backfill/backup_source.rs @@ -25,14 +25,27 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::Ordering; +use std::time::Instant; use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncReadExt}; -use tokio::sync::Mutex; use walrus::pg::replication::base_backup::Tablespace; +crate::atomic_stats! { + /// Bootstrap pump stage attribution, shared from the orchestrator down + /// through the source. Measures what the pump hands over, not what a + /// sink made of it: entry counts live on the sinks' own stats + pub struct PumpStats { + /// Body bytes handed to a `Tap` entry + pub bytes_tapped, + /// Inside `EntrySink::chunk`: page framing, decode, channel send. + /// Against `page_walk.decode_nanos` this is the tap's overhead + pub sink_chunk_nanos, + } +} + /// Filesystem-object kind. Tar-driven sources translate tar entry types /// here; the trait does not expose tar. #[derive(Debug, Clone, PartialEq, Eq)] @@ -63,24 +76,48 @@ pub struct FileMeta { } /// Sink routing decision per file at `begin()`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FileAction { /// Source writes body / materializes dir or symlink under - /// `data_dir`; no `chunk()` + /// `data_dir`; no body callbacks Keep, - /// Source drains body unread; no land, no `chunk()` + /// Source drains body unread; no land, no body callbacks Skip, - /// Source streams body through `chunk()`, no land. Dir / Symlink - /// fire `chunk()` zero times - Tap, + /// Source streams the body into this owned sink, then drops it. + /// Owned rather than a shared-state decision so concurrent entries + /// need no lock on the body path. Dir / Symlink chunk zero times + Tap(Box), } -/// Per-file token threaded through `begin`/`chunk`/`end`. Sinks key -/// per-entry state on it so concurrent entries (object_store fan-out -/// under `buffer_unordered`, sink mutex released across body reads) -/// keep independent state. Monotonic per source run, globally unique. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct EntryId(pub u64); +impl FileAction { + pub fn is_tap(&self) -> bool { + matches!(self, FileAction::Tap(_)) + } +} + +impl std::fmt::Debug for FileAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + FileAction::Keep => "Keep", + FileAction::Skip => "Skip", + FileAction::Tap(_) => "Tap", + }) + } +} + +/// One `Tap` entry's body consumer. Owned by the source task driving that +/// entry, so nothing here contends with another entry: whatever state a +/// page walk or SLRU accumulator needs lives in the impl, and shared +/// state it does touch (catalog map, counters, channel sender) is +/// already `Sync`. +#[async_trait] +pub trait EntrySink: Send { + /// Body bytes in file order, arbitrary chunk sizes. Awaiting here + /// backpressures the source + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()>; + + /// Fires once, after the last `chunk` + async fn end(self: Box) -> io::Result<()>; +} /// Mirrors wal-rus `pg::replication::base_backup::StartInfo` so callers /// wired to wal-rus types don't translate. @@ -98,44 +135,54 @@ pub struct EndInfo { pub timeline: u32, } -/// Per-file consumer, driven by [`BackupSource::run`]: `start` once, -/// then per file `begin` / (if `Tap`) zero-or-more `chunk` / `end`, -/// then `finish` once. +/// Per-file router, driven by [`BackupSource::run`]: `start` once, then +/// `begin` per file, then `finish` once. A `Tap` decision hands back its +/// own [`EntrySink`], so the body path never re-enters here. /// -/// `Send` because parallel source impls drive the sink from worker -/// tasks. All methods async so a sink can backpressure its driver -/// directly: `chunk` awaiting a full downstream channel parks the body -/// read, which parks the source fetch. No intermediate buffer. +/// `&self` throughout: routing reads immutable state and bumps atomics, so +/// parallel source impls share one `Arc` with no lock. A source that holds +/// a lock across a body read serializes every other entry behind it, +/// which is what the owned entry sink exists to avoid. #[async_trait] -pub trait BackupSink: Send { - async fn start(&mut self, _info: &StartInfo) -> io::Result<()> { +pub trait BackupSink: Send + Sync { + async fn start(&self, _info: &StartInfo) -> io::Result<()> { Ok(()) } /// Must be cheap; per-file dispatch is the hot path - async fn begin(&mut self, entry: EntryId, meta: &FileMeta) -> io::Result; + async fn begin(&self, meta: &FileMeta) -> io::Result; - /// Body bytes for a `Tap` entry, in file order per entry, arbitrary - /// chunk sizes. Calls for distinct entries may interleave. Awaiting - /// here backpressures the source. - async fn chunk(&mut self, entry: EntryId, bytes: &[u8]) -> io::Result<()>; + async fn finish(&self, _info: &EndInfo) -> io::Result<()> { + Ok(()) + } +} - /// Fires once per `begin()`, regardless of action returned - async fn end(&mut self, entry: EntryId) -> io::Result<()>; +/// Everything a tar-part driver needs behind one `Arc`, so a parallel +/// source hands its workers a clone instead of four arguments. `data_dir` +/// receives `Keep`d bodies. +pub struct PumpTarget { + pub data_dir: PathBuf, + pub sink: Arc, + pub stats: Arc, +} - async fn finish(&mut self, _info: &EndInfo) -> io::Result<()> { - Ok(()) +impl PumpTarget { + pub fn new(data_dir: PathBuf, sink: Arc, stats: Arc) -> Self { + Self { + data_dir, + sink, + stats, + } } } -/// `data_dir` receives `Keep`d bodies. Sink behind `Arc>` so -/// parallel source impls share it without per-worker copies. #[async_trait] pub trait BackupSource: Send { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> anyhow::Result<(StartInfo, EndInfo)>; } @@ -190,11 +237,9 @@ pub(crate) fn tar_entry_meta( /// Drain one tokio_tar archive against a sink, emitting per-entry /// callbacks. Called by `DirectSource` (per `BackupEvent::Archive.body`) /// and `ObjectStoreSource` (per fetched tar part). -pub(crate) async fn pump_tar_to_sink( +pub async fn pump_tar_to_sink( archive: &mut tokio_tar::Archive, - data_dir: &Path, - sink: &Arc>, - next_entry: &AtomicU64, + target: &PumpTarget, ) -> io::Result<()> where R: AsyncRead + Unpin + Send, @@ -207,35 +252,22 @@ where let Some(meta) = tar_entry_meta(&entry)? else { continue; }; - let id = EntryId(next_entry.fetch_add(1, Ordering::Relaxed)); - pump_entry(&mut entry, &meta, data_dir, sink, id).await?; + pump_entry(&mut entry, &meta, target).await?; } Ok(()) } /// One tar entry through the sink. Factored so callers can drive /// non-tar-shaped FileMeta sequences (e.g. inline symlink emission). -pub(crate) async fn pump_entry( - body: &mut R, - meta: &FileMeta, - data_dir: &Path, - sink: &Arc>, - entry: EntryId, -) -> io::Result<()> +pub async fn pump_entry(body: &mut R, meta: &FileMeta, target: &PumpTarget) -> io::Result<()> where R: AsyncRead + Unpin + ?Sized, { - let action = { - let mut s = sink.lock().await; - s.begin(entry, meta).await? - }; - match action { - FileAction::Keep => write_kept(body, meta, data_dir).await?, - FileAction::Skip => drain_to_void(body).await?, - FileAction::Tap => stream_to_sink(body, meta, sink, entry).await?, + match target.sink.begin(meta).await? { + FileAction::Keep => write_kept(body, meta, &target.data_dir).await, + FileAction::Skip => drain_to_void(body).await, + FileAction::Tap(entry) => stream_to_entry(body, meta, entry, &target.stats).await, } - let mut s = sink.lock().await; - s.end(entry).await } async fn drain_to_void(body: &mut R) -> io::Result<()> @@ -252,18 +284,18 @@ where Ok(()) } -async fn stream_to_sink( +async fn stream_to_entry( body: &mut R, meta: &FileMeta, - sink: &Arc>, - entry: EntryId, + mut entry: Box, + stats: &PumpStats, ) -> io::Result<()> where R: AsyncRead + Unpin + ?Sized, { if !matches!(meta.kind, FileKind::File) { drain_to_void(body).await?; - return Ok(()); + return entry.end().await; } let mut buf = [0u8; 64 * 1024]; loop { @@ -271,13 +303,16 @@ where if n == 0 { break; } - // Guard held across the chunk await: a full downstream channel - // parks this read (and any concurrent part contending the lock), - // propagating backpressure to the source fetch - let mut s = sink.lock().await; - s.chunk(entry, &buf[..n]).await?; + stats.bytes_tapped.fetch_add(n as u64, Ordering::Relaxed); + // Awaiting here parks only this entry's read, and through it this + // part's fetch. Other parts keep draining + let entered = Instant::now(); + entry.chunk(&buf[..n]).await?; + stats + .sink_chunk_nanos + .fetch_add(entered.elapsed().as_nanos() as u64, Ordering::Relaxed); } - Ok(()) + entry.end().await } async fn write_kept(body: &mut R, meta: &FileMeta, data_dir: &Path) -> io::Result<()> @@ -339,9 +374,7 @@ where #[allow(dead_code)] pub(crate) async fn emit_tablespace_symlink( tablespace: &Tablespace, - data_dir: &Path, - sink: &Arc>, - entry: EntryId, + target: &PumpTarget, ) -> io::Result<()> { if tablespace.is_default() { return Ok(()); @@ -355,7 +388,7 @@ pub(crate) async fn emit_tablespace_symlink( }, }; let mut body = tokio::io::empty(); - pump_entry(&mut body, &meta, data_dir, sink, entry).await + pump_entry(&mut body, &meta, target).await } #[cfg(test)] @@ -417,34 +450,59 @@ mod tests { use super::*; /// Collects every callback into `events` so tests assert on the - /// exact sequence; `tapped` holds captured Tap chunks per file. + /// exact sequence; `tapped` holds captured Tap bodies per file. #[derive(Debug, Default)] pub(crate) struct RecordingSink { - pub events: Vec, - pub tapped: Vec<(PathBuf, Vec)>, - cur_path: Option, - cur_tap: Option>, + pub events: std::sync::Mutex>, + pub tapped: std::sync::Mutex)>>, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Event { Start { start_lsn: u64, timeline: u32 }, - Begin { path: PathBuf, action: FileAction }, + Begin { path: PathBuf, action: &'static str }, Chunk { len: usize }, End { path: PathBuf }, Finish { end_lsn: u64 }, } + /// Owned per-entry recorder, so the sink itself stays lock-free on the + /// body path exactly as production sinks do + struct RecordingEntry { + sink: Arc, + path: PathBuf, + body: Vec, + } + + #[async_trait] + impl EntrySink for RecordingEntry { + async fn chunk(&mut self, bytes: &[u8]) -> io::Result<()> { + self.sink + .events + .lock() + .unwrap() + .push(Event::Chunk { len: bytes.len() }); + self.body.extend_from_slice(bytes); + Ok(()) + } + async fn end(self: Box) -> io::Result<()> { + let RecordingEntry { sink, path, body } = *self; + sink.tapped.lock().unwrap().push((path.clone(), body)); + sink.events.lock().unwrap().push(Event::End { path }); + Ok(()) + } + } + #[async_trait] - impl BackupSink for RecordingSink { - async fn start(&mut self, info: &StartInfo) -> io::Result<()> { - self.events.push(Event::Start { + impl BackupSink for Arc { + async fn start(&self, info: &StartInfo) -> io::Result<()> { + self.events.lock().unwrap().push(Event::Start { start_lsn: info.start_lsn, timeline: info.timeline, }); Ok(()) } - async fn begin(&mut self, _entry: EntryId, meta: &FileMeta) -> io::Result { + async fn begin(&self, meta: &FileMeta) -> io::Result { let s = meta.path.to_string_lossy(); let action = if s.starts_with("pg_replslot/") { if matches!(meta.kind, FileKind::Dir) { @@ -453,35 +511,31 @@ mod tests { FileAction::Skip } } else if s == "base/5/16400" { - FileAction::Tap + FileAction::Tap(Box::new(RecordingEntry { + sink: self.clone(), + path: meta.path.clone(), + body: Vec::new(), + })) } else { FileAction::Keep }; - self.events.push(Event::Begin { + self.events.lock().unwrap().push(Event::Begin { path: meta.path.clone(), - action, + action: match action { + FileAction::Keep => "Keep", + FileAction::Skip => "Skip", + FileAction::Tap(_) => "Tap", + }, }); - self.cur_path = Some(meta.path.clone()); - self.cur_tap = (action == FileAction::Tap).then(Vec::new); - Ok(action) - } - async fn chunk(&mut self, _entry: EntryId, bytes: &[u8]) -> io::Result<()> { - self.events.push(Event::Chunk { len: bytes.len() }); - if let Some(buf) = self.cur_tap.as_mut() { - buf.extend_from_slice(bytes); - } - Ok(()) - } - async fn end(&mut self, _entry: EntryId) -> io::Result<()> { - let path = self.cur_path.take().unwrap_or_default(); - if let Some(buf) = self.cur_tap.take() { - self.tapped.push((path.clone(), buf)); + if !action.is_tap() { + self.events.lock().unwrap().push(Event::End { + path: meta.path.clone(), + }); } - self.events.push(Event::End { path }); - Ok(()) + Ok(action) } - async fn finish(&mut self, info: &EndInfo) -> io::Result<()> { - self.events.push(Event::Finish { + async fn finish(&self, info: &EndInfo) -> io::Result<()> { + self.events.lock().unwrap().push(Event::Finish { end_lsn: info.end_lsn, }); Ok(()) @@ -573,13 +627,11 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let data_dir = tmp.path(); let tar_bytes = testing::build_synthetic_tar().await; - let recording = Arc::new(Mutex::new(RecordingSink::default())); - let sink: Arc> = recording.clone(); + let recording = Arc::new(RecordingSink::default()); + let sink: Arc = Arc::new(recording.clone()); let mut archive = tokio_tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let next_entry = AtomicU64::new(0); - pump_tar_to_sink(&mut archive, data_dir, &sink, &next_entry) - .await - .unwrap(); + let target = PumpTarget::new(data_dir.to_path_buf(), sink, Arc::new(PumpStats::default())); + pump_tar_to_sink(&mut archive, &target).await.unwrap(); assert!(data_dir.join("base/5/1259").exists(), "catalog must land"); assert!(data_dir.join("global/1213").exists(), "global must land"); @@ -590,10 +642,9 @@ mod tests { // User heap tapped, not landed assert!(!data_dir.join("base/5/16400").exists()); - let r = recording.lock().await; + let events = recording.events.lock().unwrap(); // Last file event must be pg_control end (contract 3) - let last_end = r - .events + let last_end = events .iter() .rev() .find_map(|e| match e { @@ -603,8 +654,7 @@ mod tests { .unwrap(); assert_eq!(last_end, PathBuf::from("pg_control")); // Tapped chunks sum to the file body length - let tapped_bytes: usize = r - .events + let tapped_bytes: usize = events .iter() .filter_map(|e| match e { Event::Chunk { len } => Some(*len), diff --git a/src/backfill/backup_source_direct.rs b/src/backfill/backup_source_direct.rs index c0d234c6..204a5ede 100644 --- a/src/backfill/backup_source_direct.rs +++ b/src/backfill/backup_source_direct.rs @@ -10,18 +10,17 @@ use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicU64; use anyhow::{Context, Result, bail}; use async_trait::async_trait; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc; use walrus::pg::replication::base_backup::{ BackupEvent, BaseBackupOpts, ChannelReader, run_base_backup, }; use walrus::pg::replication::conn::{PgConfig, ReplicationConn}; use crate::backfill::backup_source::{ - BackupSink, BackupSource, EndInfo, StartInfo, pump_tar_to_sink, + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, }; /// Replication-protocol BASE_BACKUP issuer @@ -41,7 +40,8 @@ impl BackupSource for DirectSource { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { let DirectSource { source, opts } = *self; @@ -58,9 +58,7 @@ impl BackupSource for DirectSource { let mut start: Option = None; let mut end: Option = None; - // Archives drain sequentially here; shared counter kept for - // symmetry with object_store's concurrent parts - let next_entry = AtomicU64::new(0); + let target = PumpTarget::new(data_dir, sink.clone(), stats); while let Some(ev) = rx.recv().await { let ev = ev.context("DirectSource: BASE_BACKUP event channel")?; @@ -71,10 +69,7 @@ impl BackupSource for DirectSource { timeline: s.timeline, tablespaces: s.tablespaces, }; - { - let mut g = sink.lock().await; - g.start(&s).await?; - } + sink.start(&s).await?; start = Some(s); } BackupEvent::Archive { meta, body } => { @@ -84,17 +79,14 @@ impl BackupSource for DirectSource { oid = meta.oid, "archive open", ); - drive_archive(body, &data_dir, sink.clone(), &next_entry).await?; + drive_archive(body, &target).await?; } BackupEvent::Finish(e) => { let e = EndInfo { end_lsn: e.end_lsn, timeline: e.timeline, }; - { - let mut g = sink.lock().await; - g.finish(&e).await?; - } + sink.finish(&e).await?; end = Some(e); } } @@ -115,13 +107,11 @@ impl BackupSource for DirectSource { /// SyncIoBridge / spawn_blocking. async fn drive_archive( body: mpsc::Receiver>, - data_dir: &std::path::Path, - sink: Arc>, - next_entry: &AtomicU64, + target: &PumpTarget, ) -> Result<()> { let reader = ChannelReader::new(body); let mut archive = tokio_tar::Archive::new(reader); - pump_tar_to_sink(&mut archive, data_dir, &sink, next_entry) + pump_tar_to_sink(&mut archive, target) .await .context("DirectSource: tar unpack")?; Ok(()) diff --git a/src/backfill/backup_source_object_store.rs b/src/backfill/backup_source_object_store.rs index 1a9d557c..3f6a890a 100644 --- a/src/backfill/backup_source_object_store.rs +++ b/src/backfill/backup_source_object_store.rs @@ -28,9 +28,6 @@ use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicU64; - -use tokio::sync::Mutex; use anyhow::{Context, Result, bail}; use async_trait::async_trait; @@ -45,7 +42,7 @@ use crate::backfill::backup_sentinel::build_lsn_pair; #[cfg(test)] use crate::backfill::backup_sentinel::{parse_timeline_from_name, tablespaces_from_spec}; use crate::backfill::backup_source::{ - BackupSink, BackupSource, EndInfo, StartInfo, pump_tar_to_sink, + BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, }; /// `parallelism` bounds in-flight data parts; `pg_control` always runs @@ -90,7 +87,8 @@ impl BackupSource for ObjectStoreSource { async fn run( self: Box, data_dir: PathBuf, - sink: Arc>, + sink: Arc, + stats: Arc, ) -> Result<(StartInfo, EndInfo)> { let ObjectStoreSource { settings, @@ -124,10 +122,7 @@ impl BackupSource for ObjectStoreSource { } let (start, end) = build_lsn_pair(&resolved, &sentinel)?; - { - let mut g = sink.lock().await; - g.start(&start).await?; - } + sink.start(&start).await?; let parts = list_tar_parts(&storage, &resolved).await?; if parts.is_empty() { @@ -149,10 +144,7 @@ impl BackupSource for ObjectStoreSource { "draining tar partitions" ); - // Shared counter across concurrent parts; unique EntryId per - // entry keeps interleaved begin/chunk on the shared sink mutex - // from clobbering each other's page-walk slot. - let next_entry = Arc::new(AtomicU64::new(0)); + let target = Arc::new(PumpTarget::new(data_dir, sink.clone(), stats)); // Phase A: bounded fan-out of data parts via buffer_unordered // try_collect short-circuits on the first part error. A plain @@ -161,16 +153,9 @@ impl BackupSource for ObjectStoreSource { .map(|key| { let storage = storage.clone(); let settings = settings.clone(); - let data_dir = data_dir.clone(); - let sink = sink.clone(); - let next_entry = next_entry.clone(); let spool = part_spool_dir.clone(); - async move { - unpack_one_part( - &settings, &storage, &key, &data_dir, &spool, sink, next_entry, - ) - .await - } + let target = target.clone(); + async move { unpack_one_part(&settings, &storage, &key, &spool, &target).await } }) .buffer_unordered(parallelism) .try_collect::>() @@ -179,22 +164,10 @@ impl BackupSource for ObjectStoreSource { // Phase B: pg_control barrier, single-task. wal-g emits one // control part; walk in sorted order if ever more for key in &control_parts { - unpack_one_part( - &settings, - &storage, - key, - &data_dir, - &part_spool_dir, - sink.clone(), - next_entry.clone(), - ) - .await?; + unpack_one_part(&settings, &storage, key, &part_spool_dir, &target).await?; } - { - let mut g = sink.lock().await; - g.finish(&end).await?; - } + sink.finish(&end).await?; Ok((start, end)) } } @@ -206,10 +179,8 @@ async fn unpack_one_part( settings: &Settings, storage: &DynStorage, key: &str, - data_dir: &std::path::Path, part_spool_dir: &std::path::Path, - sink: Arc>, - next_entry: Arc, + target: &PumpTarget, ) -> Result<()> { let method = method_from_key(key); let body = storage @@ -222,7 +193,7 @@ async fn unpack_one_part( let decoded = compression::decode(method, Box::pin(spooled)); let mut archive = tokio_tar::Archive::new(decoded); - pump_tar_to_sink(&mut archive, data_dir, &sink, &next_entry) + pump_tar_to_sink(&mut archive, target) .await .with_context(|| format!("ObjectStoreSource: tar unpack {key}"))?; tracing::info!( @@ -242,6 +213,13 @@ async fn unpack_one_part( /// so no error or panic path can leak it. Same trick as /// [`crate::spill::BodySpoolFile`]. Bytes are still compressed here, so /// scratch tracks object size rather than the unpacked tar. +/// +/// Per-entry tap sinks removed the shared-lock stall this was also +/// covering, and the page walk outruns any realistic GET, but the spool +/// stays: the pipeline it feeds backpressures +/// on ClickHouse by design, so a live body would be held for however long +/// CH is the limiter, which the 60s cap does not allow. Decode speed was +/// never the binding term. async fn spool_backup_part( dir: &std::path::Path, key: &str, diff --git a/src/backfill/bootstrap_oracle.rs b/src/backfill/bootstrap_oracle.rs index d4eceab8..bd85db26 100644 --- a/src/backfill/bootstrap_oracle.rs +++ b/src/backfill/bootstrap_oracle.rs @@ -13,6 +13,7 @@ use crate::column_rules::ColumnRules; use crate::emit::ch_emitter::TablePlan; use crate::mapping::{MappingSnapshot, SystemColumns}; use crate::ops::oracle::Oracle; +use crate::schema::RelName; const ORACLE_PORT: u16 = 55440; @@ -28,6 +29,7 @@ impl BootstrapOracle { source_conninfo: String, source_password: Option, bridge_lib_dir: Option, + workers: usize, connect_budget: Duration, ) -> Result { let data_dir = base_dir.join("pg"); @@ -57,6 +59,7 @@ impl BootstrapOracle { let mut bridge = BridgeConf::in_dir(&b_sock); bridge.socket_path = b_bridge; bridge.library_dir = bridge_lib_dir; + bridge.workers = workers; let cfg_b = oracle_cfg(&b_data, &b_base, &b_sock, Some(bridge)); let b = Shadow::new(cfg_b); b.write_base_conf().context("serve conf")?; @@ -67,9 +70,10 @@ impl BootstrapOracle { .context("bootstrap oracle provision task")? .context("bootstrap oracle provision")?; - let bridge = crate::ops::bridge::connect_with_budget(&bridge_socket, connect_budget) - .await - .context("bootstrap oracle bridge connect")?; + let bridge = + crate::ops::bridge::connect_with_budget(&bridge_socket, workers, connect_budget) + .await + .context("bootstrap oracle bridge connect")?; Ok(Self { shadow, oracle: Arc::new(Oracle::new(Arc::new(bridge))), @@ -124,19 +128,28 @@ fn run_pg_dump(conninfo: &str, password: Option<&str>) -> Result { String::from_utf8(out.stdout).context("pg_dump output not utf8") } +/// Provisioning costs an `initdb` + `pg_dump` + apply + restart, so gate it +/// on the relations that actually reach ClickHouse. `walked` narrows the +/// mapped set further to what the greenfield snapshot page-walks: an +/// `initial_load = "none"` relation ships no bootstrap row, so its oracle +/// columns are not a reason to stand up a side Postgres pub fn needs_oracle( catalog: &CatalogMap, tables: &MappingSnapshot, column_rules: &ColumnRules, + walked: impl Fn(&RelName) -> bool, ) -> bool { let alloc = Allocator::stdlib(); let system = SystemColumns::default(); - catalog.descriptors().any(|desc| { - tables.get(&desc.rel_name).is_some_and(|mapping| { - TablePlan::build(alloc, desc, mapping, column_rules, &system) - .map_or(true, |plan| plan.needs_oracle()) + catalog + .descriptors() + .filter(|d| walked(&d.rel_name)) + .any(|desc| { + tables.get(&desc.rel_name).is_some_and(|mapping| { + TablePlan::build(alloc, desc, mapping, column_rules, &system) + .map_or(true, |plan| plan.needs_oracle()) + }) }) - }) } #[cfg(test)] @@ -230,7 +243,7 @@ mod tests { "Nullable(JSON)", "premise: default bridge maps json to a composite CH target", ); - assert!(needs_oracle(&catalog, &tables, &rules)); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] @@ -243,7 +256,7 @@ mod tests { ]), &rules, ); - assert!(!needs_oracle(&catalog, &tables, &rules)); + assert!(!needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] @@ -256,13 +269,27 @@ mod tests { ]), &rules, ); - assert!(needs_oracle(&catalog, &tables, &rules)); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); } #[test] fn unmapped_relation_needs_no_oracle() { let rules = ColumnRules::default(); let (catalog, _) = bridged(rel(vec![attr(1, "doc", JSONOID, "json", -1)]), &rules); - assert!(!needs_oracle(&catalog, &Arc::default(), &rules)); + assert!(!needs_oracle(&catalog, &Arc::default(), &rules, |_| true)); + } + + #[test] + fn relation_out_of_the_snapshot_needs_no_oracle() { + let rules = ColumnRules::default(); + let (catalog, tables) = bridged( + rel(vec![ + attr(1, "id", INT4OID, "int4", 4), + attr(2, "doc", JSONOID, "json", -1), + ]), + &rules, + ); + assert!(needs_oracle(&catalog, &tables, &rules, |_| true)); + assert!(!needs_oracle(&catalog, &tables, &rules, |_| false)); } } diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 1aafc293..69871dea 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -84,6 +84,13 @@ use crate::schema::{ }; use crate::source::source_feed::open_sql_client; use crate::toast::ToastResolver; + +/// Rows per COPY-backfill channel hop. Byte trigger below bounds the wide-row +/// case, so this only caps the narrow-row hop rate +const COPY_SLAB_ROWS: usize = 1024; +/// Decoded value bytes per hop. With +/// [`BOOTSTRAP_TUPLE_CHANNEL_CAP`] this is the resident payload ceiling +const COPY_SLAB_BYTES: usize = 1 << 20; use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; const LEDGER_FILENAME: &str = "backfills.toml"; @@ -1071,7 +1078,7 @@ impl CopyBackfiller { let mut catalog = CatalogMap::new(); catalog.insert(desc.clone()); - let (tup_tx, tup_rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let (tup_tx, tup_rx) = mpsc::channel::>(BOOTSTRAP_TUPLE_CHANNEL_CAP); let drain = tokio::spawn(bootstrap::drain( tup_rx, catalog, @@ -1101,6 +1108,11 @@ impl CopyBackfiller { let stream = BinaryCopyOutStream::new(copy, &byte_fields); futures::pin_mut!(stream); let mut rows = 0u64; + // Slab the channel: one hop per `COPY_SLAB_BYTES` of decoded + // values, not one per row. Byte-triggered so wide rows keep the + // resident bound whatever the row count + let mut slab: Vec = Vec::new(); + let mut slab_bytes = 0usize; while let Some(row) = stream.next().await { let row = row.context("backfill: COPY stream")?; let mut columns: Vec> = vec![None; plan.natts]; @@ -1113,22 +1125,37 @@ impl CopyBackfiller { .unwrap_or(ColumnValue::Null); columns[(cp.attnum - 1).max(0) as usize] = Some(v); } + slab_bytes += columns + .iter() + .flatten() + .map(ColumnValue::approx_bytes) + .sum::(); + slab.push(BackfillTuple { + rfn: desc.rfn, + xid: 0, + xmax: 0, + infomask: 0, + source_lsn: s_lsn.get(), + // COPY text rows have no on-page TID (values arrive + // detoasted, no chunks flow here) + blkno: 0, + offnum: 0, + columns, + }); + rows += 1; + if slab.len() >= COPY_SLAB_ROWS || slab_bytes >= COPY_SLAB_BYTES { + slab_bytes = 0; + tup_tx + .send(std::mem::take(&mut slab)) + .await + .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; + } + } + if !slab.is_empty() { tup_tx - .send(BackfillTuple { - rfn: desc.rfn, - xid: 0, - xmax: 0, - infomask: 0, - source_lsn: s_lsn.get(), - // COPY text rows have no on-page TID (values arrive - // detoasted, no chunks flow here) - blkno: 0, - offnum: 0, - columns, - }) + .send(slab) .await .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; - rows += 1; } rows }; diff --git a/src/bin/stream.rs b/src/bin/stream.rs index b4b750d2..19d2fddf 100644 --- a/src/bin/stream.rs +++ b/src/bin/stream.rs @@ -49,7 +49,8 @@ use walrus::pg::replication::base_backup::BaseBackupOpts; use walrus::pg::replication::conn::PgConfig; use walrus::pg::replication::tls::SslMode; use walshadow::backfill_bootstrap::{ - BootstrapConfig, BootstrapOutcome, drain_backfill, seed_in_snapshot, spawn_greenfield_bootstrap, + BootstrapConfig, BootstrapOutcome, BootstrapProgress, drain_backfill, seed_in_snapshot, + spawn_greenfield_bootstrap, }; use walshadow::backup_source::BackupSource; use walshadow::backup_source_direct::DirectSource; @@ -385,6 +386,12 @@ struct Args { /// `dynamic_library_path`. #[arg(long)] bridge_lib_dir: Option, + /// Bridge workers to run on a daemon-owned shadow, and sockets to pool + /// against. Each worker serves one request at a time, so a value below + /// the decode pool's width caps oracle throughput at that many + /// concurrent round trips whatever the daemon does. + #[arg(long, default_value_t = 1)] + bridge_workers: usize, /// Walsender bind address. `127.0.0.1:0` lets the kernel pick a free /// port, valid only for externally managed shadow (no /// `--bootstrap-shadow-data-dir`): operator reads @@ -937,6 +944,7 @@ async fn run_session( .with_context(|| format!("ensure physical replication slot {slot}"))?; tracing::info!(target: "walshadow", slot, "physical replication slot ready"); } + let mut bootstrap_progress: Option = None; let bootstrap_end_lsn: Option = if matches!(shadow_start, ShadowStart::Bootstrap(_)) { if !args.skip_preflight { let source_sql = feed @@ -952,11 +960,18 @@ async fn run_session( .into_result() .context("pre-flight rejected bootstrap")?; } - Some( - run_bootstrap(&cfg, &mut feed, args, &bootstrap_plan, ch_config.clone()) - .await - .context("bootstrap")?, + let (end_lsn, progress) = run_bootstrap( + &cfg, + &mut feed, + args, + &bootstrap_plan, + ch_config.clone(), + &metrics, ) + .await + .context("bootstrap")?; + bootstrap_progress = Some(progress); + Some(end_lsn) } else { None }; @@ -1242,7 +1257,7 @@ async fn run_session( let connect_budget = Duration::from_secs(args.shadow_connect_timeout); let bridge_path = args.bridge_socket_path(); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(&bridge_path, connect_budget) + walshadow::bridge::connect_with_budget(&bridge_path, args.bridge_workers, connect_budget) .await .with_context(|| format!("connect bridge at {}", bridge_path.display()))?, ); @@ -1250,6 +1265,7 @@ async fn run_session( tracing::info!( target: "walshadow::bridge", socket = %bridge_path.display(), + workers = bridge.pool_size(), pg_version = info.map(|i| i.pg_version_num).unwrap_or(0), in_recovery = info.map(|i| i.in_recovery).unwrap_or(false), "bridge connected", @@ -1724,6 +1740,7 @@ async fn run_session( addr = %addr, decoders, inserters, + resolvers = bridge.pool_size(), "parallel decode+insert pipeline starting", ); PipelineConfig { @@ -2667,6 +2684,7 @@ async fn run_session( &desc_log, metrics_resolver.as_deref(), metrics_backfiller.as_deref(), + bootstrap_progress.as_ref(), ) .await; if advanced { @@ -3313,6 +3331,7 @@ async fn populate_metrics( desc_log: &walshadow::desc_log::DescriptorLog, config_resolver: Option<&ConfigResolver>, backfiller: Option<&walshadow::copy_backfill::CopyBackfiller>, + bootstrap: Option<&BootstrapProgress>, ) { use std::collections::BTreeMap; use walshadow::record::rmgr_label; @@ -3482,6 +3501,15 @@ async fn populate_metrics( inserter_batches_in_total: emitter_stats .map(|s| s.inserter_batches_in.load(Ordering::Relaxed)) .unwrap_or(0), + inserter_ch_seconds_total: emitter_stats + .map(|s| s.inserter_ch_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), + inserter_encode_seconds_total: emitter_stats + .map(|s| s.inserter_encode_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), + oracle_resolve_seconds_total: emitter_stats + .map(|s| s.oracle_resolve_nanos.load(Ordering::Relaxed) as f64 / 1e9) + .unwrap_or(0.0), process_cpu_seconds_total: proc_cpu, process_resident_memory_bytes: proc_rss, emitter_xacts_total: emitter_stats @@ -3493,6 +3521,9 @@ async fn populate_metrics( emitter_deletes_discarded: emitter_stats .map(|s| s.deletes_discarded.load(Ordering::Relaxed)) .unwrap_or(0), + oracle_local_columns_total: emitter_stats + .map(|s| s.oracle_local_columns.load(Ordering::Relaxed)) + .unwrap_or(0), oracle_blocks_total: oracle_stats .map(|s| s.blocks.load(Ordering::Relaxed)) .unwrap_or(0), @@ -3585,6 +3616,10 @@ async fn populate_metrics( bridge_requests_by_op: bridge_ops(bridge_stats.map(|b| &b.requests)), bridge_errors_by_op: bridge_ops(bridge_stats.map(|b| &b.errors)), bridge_request_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.request_nanos)), + bridge_lock_wait_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.lock_wait_nanos)), + bridge_service_nanos_by_op: bridge_ops(bridge_stats.map(|b| &b.service_nanos)), + bridge_request_bytes_by_op: bridge_ops(bridge_stats.map(|b| &b.request_bytes)), + bridge_response_bytes_by_op: bridge_ops(bridge_stats.map(|b| &b.response_bytes)), bridge_reconnects_total: bridge_gauge(bridge_stats, |b| &b.reconnects), bridge_scan_rows_total: bridge_gauge(bridge_stats, |b| &b.scan_rows), bridge_scan_replay_moved_total: bridge_gauge(bridge_stats, |b| &b.scan_replay_moved), @@ -3592,10 +3627,32 @@ async fn populate_metrics( &b.scan_subtrans_mismatch }), bridge_native_bytes_total: bridge_gauge(bridge_stats, |b| &b.native_bytes), + ..bootstrap_gauges(bootstrap) }; registry.set(snap).await; } +/// Bootstrap stage attribution, frozen at its final values once the pump +/// returns. Rendered for the whole session so a slow initial load stays +/// attributable after the fact +fn bootstrap_gauges(progress: Option<&BootstrapProgress>) -> MetricsSnapshot { + let Some(p) = progress else { + return MetricsSnapshot::default(); + }; + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed); + MetricsSnapshot { + bootstrap_bytes_tapped: ld(&p.pump.bytes_tapped), + bootstrap_pages_walked: ld(&p.page_walk.pages_walked), + bootstrap_tuples_emitted: ld(&p.page_walk.tuples_emitted), + bootstrap_files_walked: ld(&p.page_walk.files_walked), + bootstrap_files_skipped_unmapped: ld(&p.page_walk.files_skipped_unmapped), + bootstrap_decode_seconds: ld(&p.page_walk.decode_nanos) as f64 / 1e9, + bootstrap_tap_seconds: ld(&p.pump.sink_chunk_nanos) as f64 / 1e9, + bootstrap_channel_block_seconds: ld(&p.page_walk.channel_block_nanos) as f64 / 1e9, + ..MetricsSnapshot::default() + } +} + /// Zero when bridge stats are unavailable, so series stays present fn bridge_gauge( stats: Option<&walshadow::bridge::BridgeStats>, @@ -4168,7 +4225,9 @@ async fn run_bootstrap( args: &Args, plan: &BootstrapPlan, ch_config: Option, -) -> Result { + metrics: &MetricsRegistry, +) -> Result<(u64, BootstrapProgress)> { + let bootstrap_started = Instant::now(); let shadow_data_dir = args .bootstrap_shadow_data_dir .clone() @@ -4284,19 +4343,114 @@ async fn run_bootstrap( let (mapping, resolved) = bootstrap_build_mapping(&emitter_cfg, &drain_catalog, args) .await .context("bootstrap: build mapping")?; - Some((emitter_cfg, mapping, resolved)) + // `initial_load = "none"` (table override, else namespace) opts a + // relation out of the greenfield snapshot: create it + stream CDC, + // but don't page-walk its existing rows. + let skip_initial: std::collections::HashSet<_> = drain_catalog + .descriptors() + .filter_map(|d| { + let rn = &d.rel_name; + let none = match emitter_cfg.table_initial_loads.get(rn) { + Some(s) => s.parse::() == Ok(InitialLoadMode::None), + None => { + resolved + .namespaces + .get(rn.namespace.as_ref()) + .and_then(|n| n.initial_load) + == Some(InitialLoadMode::None) + } + }; + none.then(|| rn.clone()) + }) + .collect(); + Some((emitter_cfg, mapping, resolved, skip_initial)) } None => None, }; + // Decline unmapped relations at `begin` so their pages never decode. + // Metrics-only (no CH) has no mapping to filter against, so it walks all + let (tap_filenodes, needs_oracle) = match &ch_target { + Some((_, mapping, resolved, skip_initial)) => { + let mapped = mapping.snapshot().await; + let walked = |rn: &RelName| mapped.contains_key(rn) && !skip_initial.contains(rn); + ( + walshadow::backfill_bootstrap::tap_filenode_set(&drain_catalog, walked) + .map(Arc::new), + walshadow::backfill::bootstrap_oracle::needs_oracle( + &drain_catalog, + &mapped, + &resolved.column_rules, + walked, + ), + ) + } + None => (None, false), + }; + + // Off the backup window: provisioning is an initdb + pg_dump + apply + + // restart, and doing it after `BASE_BACKUP` opens parks a live backup + // through all of it — in object-store mode against walrus's 60 s request + // cap + let bootstrap_oracle = if needs_oracle { + let source_conninfo = format!( + "host={} port={} user={} dbname={} sslmode={}", + src_cfg.host, + src_cfg.port, + src_cfg.user, + src_cfg.database, + if src_cfg.sslmode == SslMode::Disable { + "disable" + } else { + "prefer" + }, + ); + Some( + walshadow::backfill::bootstrap_oracle::BootstrapOracle::provision( + args.spill_dir.join("bootstrap_oracle"), + source_conninfo, + src_cfg.password.clone(), + args.bridge_lib_dir.clone(), + args.bridge_workers, + Duration::from_secs(args.shadow_connect_timeout), + ) + .await + .context( + "bootstrap oracle: greenfield needs it to resolve tier-3 types; \ + refusing to load empty columns", + )?, + ) + } else { + None + }; + let oracle = bootstrap_oracle.as_ref().map(|o| o.oracle()); + prepare_bootstrap_dir(&shadow_data_dir) .await .context("prepare shadow data dir for bootstrap")?; - let cfg = BootstrapConfig::new(shadow_data_dir.clone()); + let mut cfg = BootstrapConfig::new(shadow_data_dir.clone()); + if let Some(set) = tap_filenodes { + cfg = cfg.with_tap_filenodes(set); + } + let progress = cfg.progress.clone(); + // Only writer of the registry until the status loop starts, so a plain + // gauge-only snapshot is the whole surface here + let ticker = tokio::spawn({ + let metrics = metrics.clone(); + let progress = progress.clone(); + async move { + let mut tick = tokio::time::interval(Duration::from_secs(5)); + loop { + tick.tick().await; + metrics.set(bootstrap_gauges(Some(&progress))).await; + } + } + }); let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); - let (shipped, outcome) = if let Some((emitter_cfg, mapping, resolved)) = ch_target { + let (shipped, outcome) = if let Some((emitter_cfg, mapping, resolved, skip_initial)) = ch_target + { // Route bootstrap rows through the shared insert tail. Bootstrap // is the easy case: every row op=Insert at _lsn = start_lsn, no // aborts / TRUNCATE / DDL. Keep operator's flush_timeout; tail @@ -4310,41 +4464,6 @@ async fn run_bootstrap( let fatal = Fatal::new(); let inserter_pool_size = emitter_cfg.inserter_pool_size; - let source_conninfo = format!( - "host={} port={} user={} dbname={} sslmode={}", - src_cfg.host, - src_cfg.port, - src_cfg.user, - src_cfg.database, - if src_cfg.sslmode == SslMode::Disable { - "disable" - } else { - "prefer" - }, - ); - let bootstrap_oracle = if walshadow::backfill::bootstrap_oracle::needs_oracle( - &drain_catalog, - &mapping.snapshot().await, - &resolved.column_rules, - ) { - Some( - walshadow::backfill::bootstrap_oracle::BootstrapOracle::provision( - args.spill_dir.join("bootstrap_oracle"), - source_conninfo, - src_cfg.password.clone(), - args.bridge_lib_dir.clone(), - Duration::from_secs(args.shadow_connect_timeout), - ) - .await - .context( - "bootstrap oracle: greenfield needs it to convert oracle columns; \ - refusing to load empty columns", - )?, - ) - } else { - None - }; - let (msg_tx, ack, tail) = tail::spawn_with_config( &emitter_cfg, inserter_pool_size, @@ -4352,7 +4471,7 @@ async fn run_bootstrap( emitter_ack, fatal.clone(), None, - bootstrap_oracle.as_ref().map(|o| o.oracle()), + oracle, ) .await .context("bootstrap: spawn insert tail")?; @@ -4363,27 +4482,6 @@ async fn run_bootstrap( "bootstrap insert tail started", ); - // `initial_load = "none"` (table override, else namespace) opts a - // relation out of the greenfield snapshot: create it + stream CDC, but - // don't page-walk its existing rows. - let skip_initial: std::collections::HashSet<_> = drain_catalog - .descriptors() - .filter_map(|d| { - let rn = &d.rel_name; - let none = match emitter_cfg.table_initial_loads.get(rn) { - Some(s) => s.parse::() == Ok(InitialLoadMode::None), - None => { - resolved - .namespaces - .get(rn.namespace.as_ref()) - .and_then(|n| n.initial_load) - == Some(InitialLoadMode::None) - } - }; - none.then(|| rn.clone()) - }) - .collect(); - let deferred_path = args.spill_dir.join("bootstrap_deferred.bin"); tokio::fs::remove_file(&deferred_path).await.ok(); let drain = tokio::spawn(bootstrap::drain( @@ -4437,18 +4535,33 @@ async fn run_bootstrap( (shipped, outcome) }; + let ld = |a: &std::sync::atomic::AtomicU64| a.load(Ordering::Relaxed); tracing::info!( target: "walshadow::bootstrap", start_lsn = format_pg_lsn(outcome.start.start_lsn).to_string(), end_lsn = format_pg_lsn(outcome.end.end_lsn).to_string(), timeline = outcome.start.timeline, - kept_files = outcome.disk.kept_files, - skipped_denylist = outcome.disk.skipped_denylist, - files_walked = outcome.page_walk.files_walked, - tuples_emitted = outcome.page_walk.tuples_emitted, + kept_files = ld(&outcome.disk.kept_files), + skipped_denylist = ld(&outcome.disk.skipped_denylist), + files_walked = ld(&outcome.page_walk.files_walked), + tuples_emitted = ld(&outcome.page_walk.tuples_emitted), drained = shipped, "bootstrap landed", ); + // Stage attribution: which of tap, decode or emitter drain owned the + // wall clock. Sum exceeds elapsed under source parallelism + tracing::info!( + target: "walshadow::bootstrap", + elapsed_secs = bootstrap_started.elapsed().as_secs_f64(), + bytes_tapped = ld(&outcome.pump.bytes_tapped), + pages_walked = ld(&outcome.page_walk.pages_walked), + tap_secs = ld(&outcome.pump.sink_chunk_nanos) as f64 / 1e9, + decode_secs = ld(&outcome.page_walk.decode_nanos) as f64 / 1e9, + channel_block_secs = ld(&outcome.page_walk.channel_block_nanos) as f64 / 1e9, + files_skipped_unmapped = ld(&outcome.page_walk.files_skipped_unmapped), + "bootstrap stage timings", + ); + ticker.abort(); if let Some((settings, storage)) = wal_hydrate { fetch_wal_into_pg_wal( @@ -4479,7 +4592,7 @@ async fn run_bootstrap( .await .context("clear completed bootstrap marker")?; - Ok(outcome.end.end_lsn) + Ok((outcome.end.end_lsn, progress)) } /// Routing map for the bootstrap drain: explicit `[table.*]` seeded up front, @@ -4639,6 +4752,7 @@ fn build_owned_shadow(args: &Args, data_dir: PathBuf) -> Shadow { let mut bridge = walshadow::shadow::BridgeConf::in_dir(&cfg.socket_dir); bridge.socket_path = args.bridge_socket_path(); bridge.library_dir = args.bridge_lib_dir.clone(); + bridge.workers = args.bridge_workers; cfg.bridge = Some(bridge); Shadow::new(cfg) } diff --git a/src/catalog/shadow.rs b/src/catalog/shadow.rs index 2d5825cf..0d8f0c0c 100644 --- a/src/catalog/shadow.rs +++ b/src/catalog/shadow.rs @@ -69,6 +69,10 @@ pub struct BridgeConf { /// Bounds a catalog lock the worker cannot get, which would otherwise hang /// against recovery pub lock_timeout: Duration, + /// `walshadow.bridge_workers`. Each worker serves one request at a time, + /// so this is how many oracle round trips can be in flight. Worker 0 + /// keeps `socket_path`; worker `i` listens on `socket_path.i` + pub workers: usize, } impl BridgeConf { @@ -79,6 +83,7 @@ impl BridgeConf { library_dir: None, io_timeout: Duration::from_secs(30), lock_timeout: Duration::from_secs(1), + workers: 1, } } @@ -99,11 +104,14 @@ impl BridgeConf { "walshadow.socket_path = '{}'\n\ walshadow.database = '{}'\n\ walshadow.io_timeout_ms = {}\n\ - walshadow.lock_timeout_ms = {}\n", + walshadow.lock_timeout_ms = {}\n\ + walshadow.bridge_workers = {}\n", quote_path(&self.socket_path), quote(dbname), self.io_timeout.as_millis(), self.lock_timeout.as_millis(), + self.workers + .clamp(1, crate::ops::bridge::MAX_BRIDGE_WORKERS), )); out } diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 848404a3..a56aa5e0 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -1486,6 +1486,51 @@ fn name_column(mut e: EmitterError, target: &str) -> EmitterError { } /// Build stable leaves before roots borrow them +/// Answer an oracle-routed column locally when the request would be a +/// no-op: every cell is bytes the daemon rendered, and the worker's `String` +/// path wraps each in a `text` that pgch casts straight back. Defaults take +/// the shape `ws_append_default` gives them — NULL under `Nullable`, empty +/// string otherwise. `None` leaves the column in the request +pub(crate) fn literal_column( + buf: &OracleColumnBuf, + target_type: &str, + n_rows: usize, +) -> Option { + if !buf.literal_only() || buf.cells().len() != n_rows { + return None; + } + // `canonical_type` renders these two exactly; anything wrapping a String + // (`LowCardinality`, `Array`) is PG's to build + let nullable = target_type == "Nullable(String)"; + if !nullable && target_type != "String" { + return None; + } + let mut offsets = Vec::with_capacity(n_rows); + let mut data = Vec::new(); + let mut null_map = vec![0u8; if nullable { n_rows } else { 0 }]; + for (i, cell) in buf.cells().iter().enumerate() { + match cell { + OracleCell::Literal(b) => data.extend_from_slice(b), + // `literal_only` admits nothing else + _ => { + if nullable { + null_map[i] = 1; + } + } + } + offsets.push(data.len() as u64); + } + Some(if nullable { + ColumnBuf::NullableString { + offsets, + data, + null_map, + } + } else { + ColumnBuf::String { offsets, data } + }) +} + pub(crate) fn build_leaf( buf: &ColumnBuf, n_rows: usize, @@ -1991,6 +2036,19 @@ crate::atomic_stats! { pub insertbatch_rows_in, pub insertbatch_batches_out, pub inserter_batches_in, + /// Inserter time inside the INSERT round trip (`send_query` through + /// `EndOfStream`), retries included. Against `inserter_pool_size × + /// elapsed` this is CH-side utilization + pub inserter_ch_nanos, + /// Inserter time rebuilding the Native block over the batch's slabs + pub inserter_encode_nanos, + /// Resolver time inside the oracle round trip, retries included. + /// Overlaps the inserters' ClickHouse time, so against + /// `inserter_ch_nanos` it says which stage is the limiter + pub oracle_resolve_nanos, + /// Oracle-routed columns the daemon built itself: rendered cells + /// against a `String` target, which PG would hand straight back + pub oracle_local_columns, } } @@ -2352,6 +2410,30 @@ mod tests { } } + /// A default under a bare `String` is the empty string, matching + /// `ws_append_default`; a target that wraps String is PG's to build + #[test] + fn literal_column_takes_only_a_bare_string_target() { + let mut buf = OracleColumnBuf::new(0, -1); + buf.push(OracleCell::Literal(b"a".to_vec())); + buf.push(OracleCell::Default); + match literal_column(&buf, "String", 2) { + Some(ColumnBuf::String { offsets, data }) => { + assert_eq!(data, b"a"); + assert_eq!(offsets, [1, 1]); + } + other => panic!("got {other:?}"), + } + for reject in ["Array(String)", "LowCardinality(String)", "JSON", "Int32"] { + assert!( + literal_column(&buf, reject, 2).is_none(), + "{reject} needs the worker", + ); + } + // Cell count must match the batch, else offsets would not + assert!(literal_column(&buf, "String", 3).is_none()); + } + #[test] fn new_for_ast_picks_shape_from_chc_type_kind() { let alloc = Allocator::stdlib(); diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 786c1485..8cdf891f 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -15,7 +15,8 @@ use crate::config::ResolvedConfig; use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::ack::AckHandle; -use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow}; +use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow, RowChunk}; +use crate::emit::pipeline::decode::DECODE_CHUNK_BYTES; use crate::emit::route::{RouteSnapshot, RowPolicy}; use crate::mapping::{MappingHandle, TableMapping}; use crate::ops::oracle::render_ext_columns; @@ -27,6 +28,11 @@ use crate::toast::{ use ahash::HashMap; use std::collections::HashSet; +/// Rows per `BatcherMsg::Rows` from the bootstrap drain. Fixed rather than +/// config-driven: bootstrap rows are uniform inserts, and the byte trigger +/// covers the fat-row case +const DRAIN_CHUNK_ROWS: usize = 1024; + /// Completion frontier for `FlushAll` and resume advance #[derive(Debug, Clone, Copy, Default)] pub struct BootstrapDrainOutcome { @@ -40,7 +46,7 @@ pub struct BootstrapDrainOutcome { /// descriptor and mapping re-resolve from frozen per-pass snapshots #[allow(clippy::too_many_arguments)] pub async fn drain( - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver>, catalog: CatalogMap, mapping_handle: MappingHandle, msg_tx: mpsc::Sender, @@ -74,79 +80,84 @@ pub async fn drain( let mut chunk_batch: Vec = Vec::new(); let mut chunk_batch_bytes = 0usize; let mut start_lsn = 0u64; - while let Some(tuple) = rx.recv().await { - let rfn = tuple.rfn; - let source_lsn = tuple.source_lsn; - start_lsn = source_lsn; - - let same = matches!(&open, Some((r, _, _)) if *r == rfn); - let seq = if same { - open.as_ref().expect("same implies open").1 - } else { - if let Some((_, prev_seq, prev_rows)) = open.take() { - ack.placed(prev_seq, prev_rows); - } - let s = next_seq; - next_seq += 1; - ack.register(s, source_lsn); - open = Some((rfn, s, 0)); - s - }; + // Rows coalesce into `BatcherMsg::Rows` on the same dual trigger the + // decode pool uses, so a walk slab costs one channel hop, not one per row + let mut out = RowBuf::default(); + while let Some(slab) = rx.recv().await { + for tuple in slab { + let rfn = tuple.rfn; + let source_lsn = tuple.source_lsn; + start_lsn = source_lsn; + + let same = matches!(&open, Some((r, _, _)) if *r == rfn); + let seq = if same { + open.as_ref().expect("same implies open").1 + } else { + if let Some((_, prev_seq, prev_rows)) = open.take() { + // Every row of the closing seq on the channel before its + // expected count is published + out.flush(&msg_tx).await?; + ack.placed(prev_seq, prev_rows); + } + let s = next_seq; + next_seq += 1; + ack.register(s, source_lsn); + open = Some((rfn, s, 0)); + s + }; - let Some(rel) = catalog.get(rfn.db_node, rfn.rel_node) else { - stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); - continue; - }; + let Some(rel) = catalog.get(rfn.db_node, rfn.rel_node) else { + stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); + continue; + }; - if skip_initial.contains(&rel.rel_name) { - continue; - } + if skip_initial.contains(&rel.rel_name) { + continue; + } - if catalog.is_toast(rfn.db_node, rfn.rel_node) { - if let Some(row) = row_from_columns(tuple, rel.oid) { - chunk_batch_bytes += row.chunk_data.len(); - chunk_batch.push(row); - if chunk_batch.len() >= CHUNK_PUT_BATCH || chunk_batch_bytes >= CHUNK_PUT_BYTES { - flush_chunks(&resolver, &mut chunk_batch).await?; - chunk_batch_bytes = 0; + if catalog.is_toast(rfn.db_node, rfn.rel_node) { + if let Some(row) = row_from_columns(tuple, rel.oid) { + chunk_batch_bytes += row.chunk_data.len(); + chunk_batch.push(row); + if chunk_batch.len() >= CHUNK_PUT_BATCH || chunk_batch_bytes >= CHUNK_PUT_BYTES + { + flush_chunks(&resolver, &mut chunk_batch).await?; + chunk_batch_bytes = 0; + } } + continue; } - continue; - } - - let Some(route) = routes.get(&rel.rel_name).cloned() else { - stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); - continue; - }; - if has_mapped_external_toast(&tuple, &route.mapping) { - if resolver.stores_chunks() { - deferred - .push(tuple) - .await - .map_err(|e| format!("bootstrap: deferred spool: {e}"))?; - stats - .bootstrap_deferred_bytes - .store(deferred.resident_bytes() as u64, Ordering::Relaxed); - stats - .bootstrap_deferred_spool_bytes - .store(deferred.spooled_bytes(), Ordering::Relaxed); + let Some(route) = routes.get(&rel.rel_name).cloned() else { + stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); continue; - } + }; + let mut tuple = tuple; - let permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; + let mut permit = None; + if has_mapped_external_toast(&tuple, &route.mapping) { + if resolver.stores_chunks() { + deferred + .push(tuple) + .await + .map_err(|e| format!("bootstrap: deferred spool: {e}"))?; + stats + .bootstrap_deferred_bytes + .store(deferred.resident_bytes() as u64, Ordering::Relaxed); + stats + .bootstrap_deferred_spool_bytes + .store(deferred.spooled_bytes(), Ordering::Relaxed); + continue; + } + permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; + } render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, permit).await?; + out.push(&msg_tx, seq, rel, route, tuple, permit).await?; bump(&mut open, &mut rows_routed); - continue; } - - let mut tuple = tuple; - render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, None).await?; - bump(&mut open, &mut rows_routed); } + out.flush(&msg_tx).await?; if let Some((_, seq, rows)) = open.take() { ack.placed(seq, rows); } @@ -185,10 +196,11 @@ pub async fn drain( }; let permit = resolve_or_fill_toast(&mut tuple, &rel, &route.mapping, &resolver).await?; render_ext_columns(&rel.attributes, &mut tuple.columns); - route_row(&msg_tx, seq, rel, route, tuple, permit).await?; + out.push(&msg_tx, seq, rel, route, tuple, permit).await?; placed += 1; rows_routed += 1; } + out.flush(&msg_tx).await?; replay .finish() .await @@ -213,25 +225,54 @@ fn bump(open: &mut Option<(walrus::pg::walparser::RelFileNode, u64, u64)>, rows_ *rows_routed += 1; } -async fn route_row( - msg_tx: &mpsc::Sender, - seq: u64, - rel: Arc, - route: Arc, - tuple: BackfillTuple, - value_permit: Option, -) -> Result<(), String> { - let committed = tuple.into_committed_insert(); - msg_tx - .send(BatcherMsg::Row(RoutedRow { +/// Coalesces routed rows into one `BatcherMsg::Rows` per +/// [`DECODE_CHUNK_BYTES`]-shaped trigger, the same amortization the streaming +/// decode pool gets. Rows of different seqs may share a chunk; the batcher +/// routes each independently +#[derive(Default)] +struct RowBuf { + rows: Vec, + bytes: usize, +} + +impl RowBuf { + async fn push( + &mut self, + msg_tx: &mpsc::Sender, + seq: u64, + rel: Arc, + route: Arc, + tuple: BackfillTuple, + value_permit: Option, + ) -> Result<(), String> { + let committed = tuple.into_committed_insert(); + self.bytes += committed.decoded.approx_bytes(); + self.rows.push(RoutedRow { seq, rel, route, committed, value_permit: value_permit.map(Arc::new), - })) - .await - .map_err(|_| "bootstrap: batcher channel closed".to_string()) + }); + if self.rows.len() >= DRAIN_CHUNK_ROWS || self.bytes >= DECODE_CHUNK_BYTES { + self.flush(msg_tx).await?; + } + Ok(()) + } + + async fn flush(&mut self, msg_tx: &mpsc::Sender) -> Result<(), String> { + if self.rows.is_empty() { + return Ok(()); + } + self.bytes = 0; + msg_tx + .send(BatcherMsg::Rows(RowChunk { + rows: std::mem::take(&mut self.rows), + permit: None, + })) + .await + .map_err(|_| "bootstrap: batcher channel closed".to_string()) + } } /// Check only columns routed to ClickHouse @@ -348,6 +389,21 @@ fn row_from_columns(mut tuple: BackfillTuple, toast_relid: u32) -> Option) -> Vec { + let mut rows = Vec::new(); + while let Some(msg) = rx.recv().await { + match msg { + BatcherMsg::Rows(chunk) => rows.extend(chunk.rows), + BatcherMsg::Row(r) => rows.push(r), + BatcherMsg::FlushAll(reply) => { + let _ = reply.send(()); + } + } + } + rows + } + use super::*; use crate::backfill::spool::DEFERRED_SPOOL_MEM_MAX; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; @@ -556,13 +612,13 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); for id in 0..3 { - tup_tx.send(tuple(16400, id)).await.unwrap(); + tup_tx.send(vec![tuple(16400, id)]).await.unwrap(); } for id in 0..2 { - tup_tx.send(tuple(16401, id)).await.unwrap(); + tup_tx.send(vec![tuple(16401, id)]).await.unwrap(); } drop(tup_tx); @@ -582,7 +638,7 @@ mod tests { )); let mut by_seq: HashMap = HashMap::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { + for r in collect_rows(&mut msg_rx).await { *by_seq.entry(r.seq).or_default() += 1; } let outcome = drain_task.await.unwrap().unwrap(); @@ -611,12 +667,12 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); // 16400, 16401(unmapped), 16400 → seqs 0,1,2; only 0 and 2 route - tup_tx.send(tuple(16400, 1)).await.unwrap(); - tup_tx.send(tuple(16401, 9)).await.unwrap(); - tup_tx.send(tuple(16400, 2)).await.unwrap(); + tup_tx.send(vec![tuple(16400, 1)]).await.unwrap(); + tup_tx.send(vec![tuple(16401, 9)]).await.unwrap(); + tup_tx.send(vec![tuple(16400, 2)]).await.unwrap(); drop(tup_tx); let stats = Arc::new(EmitterStats::default()); @@ -634,10 +690,11 @@ mod tests { std::collections::HashSet::new(), )); - let mut seqs: Vec = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - seqs.push(r.seq); - } + let seqs: Vec = collect_rows(&mut msg_rx) + .await + .iter() + .map(|r| r.seq) + .collect(); let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 3, "three distinct rfn runs"); assert_eq!(outcome.rows_routed, 2, "unmapped rel routed nothing"); @@ -660,10 +717,10 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); drop(tup_tx); @@ -684,10 +741,7 @@ mod tests { std::collections::HashSet::new(), )); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 1); assert_eq!(outcome.rows_routed, 1); @@ -719,16 +773,16 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); let spool_tmp = tempfile::tempdir().unwrap(); // toast chunk first (its own zero-row seq), then the referring main row tup_tx - .send(toast_chunk_tuple(16500, 1, 0, b"hello")) + .send(vec![toast_chunk_tuple(16500, 1, 0, b"hello")]) .await .unwrap(); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); drop(tup_tx); @@ -750,10 +804,7 @@ mod tests { std::collections::HashSet::new(), )); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 3, "toast seq, main seq, deferred seq"); assert_eq!(outcome.rows_routed, 1); @@ -782,14 +833,14 @@ mod tests { let emitter_ack = Arc::new(crate::pos::Monotone::new(0)); let (ack, collector) = ack::spawn(emitter_ack); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let (tup_tx, tup_rx) = mpsc::channel::(64); + let (tup_tx, tup_rx) = mpsc::channel::>(64); tup_tx - .send(toast_chunk_tuple(16500, 1, 0, b"hello")) + .send(vec![toast_chunk_tuple(16500, 1, 0, b"hello")]) .await .unwrap(); tup_tx - .send(bytea_toast_tuple(16400, 16500, 1)) + .send(vec![bytea_toast_tuple(16400, 16500, 1)]) .await .unwrap(); @@ -817,10 +868,7 @@ mod tests { mapping.mutate(|m| Arc::make_mut(m).clear()).await; drop(tup_tx); - let mut rows = Vec::new(); - while let Some(BatcherMsg::Row(r)) = msg_rx.recv().await { - rows.push(r); - } + let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed, 1, "unmapping must not drop the row"); assert_eq!(rows.len(), 1); diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index 22e05169..87dd3089 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -23,8 +23,8 @@ use crate::config::ResolvedConfig; use crate::emit::ch_emitter::{ColumnBuf, EmitterConfig, EmitterStats, build_leaf, build_root}; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::AckHandle; -use crate::emit::pipeline::batcher::{BatchMeta, InsertBatch}; -use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; +use crate::emit::pipeline::batcher::BatchMeta; +use crate::emit::pipeline::resolver::ResolvedBatch; use crate::schema::RelName; use ahash::{HashMap, HashMapExt}; use std::sync::Arc; @@ -46,7 +46,6 @@ struct Inserter { /// compression are re-read at each batch boundary (a compression change /// reconnects, since the codec is fixed at connect). config_rx: Option>>, - oracle: Option>, } impl Inserter { @@ -84,6 +83,7 @@ impl Inserter { let retry = self.config.retry.clone(); let mut attempt = 0u32; let mut backoff = retry.initial_backoff; + let started = std::time::Instant::now(); reconnect_if_idle(&mut self.client, &self.config, self.last_used).await?; loop { let attempt_result = with_timeout(self.config.insert_timeout, async { @@ -97,6 +97,9 @@ impl Inserter { match attempt_result { Ok(()) => { self.last_used = std::time::Instant::now(); + self.stats + .inserter_ch_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); return Ok(()); } Err(e) if is_retryable(&e) && attempt < retry.max_attempts => { @@ -110,8 +113,13 @@ impl Inserter { } } - async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { - while let Ok(batch) = rx.recv().await { + async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { + while let Ok(ResolvedBatch { + batch, + local, + resolved, + }) = rx.recv().await + { // Live emitter knobs (overlay active): pick up the retry budget and // compression. A compression change needs a fresh client — the codec // is fixed at connect — reconnected here at a batch boundary, never @@ -174,22 +182,17 @@ impl Inserter { .asts .remove(&batch.meta.table_key) .expect("ensure_asts inserted"); + // Columns the resolver built locally stand in for their oracle + // buffers; the rest still splice out of the answered block + let bufs: Vec<&ColumnBuf> = batch + .buffers + .iter() + .enumerate() + .map(|(i, buf)| local.get(i).and_then(Option::as_ref).unwrap_or(buf)) + .collect(); let result = 'send: { - // Keep resolved block alive across every INSERT retry - let resolved = match resolve_oracle_with_retry( - self.oracle.clone(), - self.alloc, - &batch, - &self.config.retry, - &self.stats, - ) - .await - { - Ok(v) => v, - Err(e) => break 'send Err(e), - }; - let leaves: Vec>> = match batch - .buffers + let encode_started = std::time::Instant::now(); + let leaves: Vec>> = match bufs .iter() .map(|buf| build_leaf(buf, batch.n_rows)) .collect::>() @@ -197,8 +200,7 @@ impl Inserter { Ok(v) => v, Err(e) => break 'send Err(e), }; - let roots: Vec>> = match batch - .buffers + let roots: Vec>> = match bufs .iter() .zip(&leaves) .map(|(buf, leaf)| match buf { @@ -234,6 +236,10 @@ impl Inserter { .map_err(Into::into) } }); + self.stats.inserter_encode_nanos.fetch_add( + encode_started.elapsed().as_nanos() as u64, + Ordering::Relaxed, + ); match appended { Ok(()) => self.send_with_retry(&batch.meta.insert_sql, &bb).await, Err(e) => Err(e), @@ -261,73 +267,15 @@ impl Inserter { } } -/// Retry transport failures only -async fn resolve_oracle_with_retry( - oracle: Option>, - alloc: Allocator, - batch: &InsertBatch, - retry: &crate::emit::ch_emitter::RetryConfig, - stats: &EmitterStats, -) -> Result, EmitterError> { - let mut attempt = 0u32; - let mut backoff = retry.initial_backoff; - loop { - match resolve_oracle(oracle.clone(), alloc, batch).await { - Ok(v) => return Ok(v), - Err(e) if e.retryable() && attempt < retry.max_attempts => { - stats.retries_attempted.fetch_add(1, Ordering::Relaxed); - attempt += 1; - backoff_step(&mut backoff, retry.max_backoff).await; - } - Err(e) => return Err(EmitterError::Type(e.to_string())), - } - } -} - -async fn resolve_oracle( - oracle: Option>, - alloc: Allocator, - batch: &InsertBatch, -) -> Result, OracleError> { - let columns: Vec> = batch - .buffers - .iter() - .enumerate() - .filter_map(|(i, buf)| match buf { - ColumnBuf::Oracle(o) => Some(OracleRequestColumn { - ordinal: i as u32, - name: &batch.meta.columns[i].name, - target_type: &batch.meta.columns[i].type_repr, - buf: o, - }), - _ => None, - }) - .collect(); - if columns.is_empty() { - return Ok(None); - } - let oracle = oracle.ok_or_else(|| { - OracleError::Absent(format!( - "{} needs the shadow oracle, which this pipeline has none of", - batch.meta.table_key - )) - })?; - oracle - .encode_batch(&columns, batch.n_rows, alloc) - .await - .map(Some) -} - pub(crate) struct PoolOptions { pub config_rx: Option>>, - pub oracle: Option>, } /// Connect `n` inserters and spawn drain loops pub(crate) async fn spawn_pool( n: usize, config: &EmitterConfig, - rx: async_channel::Receiver, + rx: async_channel::Receiver, ack: AckHandle, stats: Arc, fatal: Fatal, @@ -345,7 +293,6 @@ pub(crate) async fn spawn_pool( ack: ack.clone(), stats: stats.clone(), config_rx: options.config_rx.clone(), - oracle: options.oracle.clone(), }; let rx = rx.clone(); let fatal = fatal.clone(); diff --git a/src/emit/pipeline/mod.rs b/src/emit/pipeline/mod.rs index 3a28e75c..fcc58788 100644 --- a/src/emit/pipeline/mod.rs +++ b/src/emit/pipeline/mod.rs @@ -18,6 +18,7 @@ pub mod inserter; pub mod plan_spool; pub mod planner; pub mod reorder; +pub mod resolver; pub mod tail; use std::sync::Arc; diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs new file mode 100644 index 00000000..63785687 --- /dev/null +++ b/src/emit/pipeline/resolver.rs @@ -0,0 +1,384 @@ +//! Oracle resolve, off the inserters' critical path. +//! +//! An inserter that resolves its own batch leaves its ClickHouse connection +//! idle for the whole round trip, and the oracle idle for the whole INSERT: +//! throughput is `1 / (T_oracle + T_ch / P)` where it could be +//! `1 / max(T_oracle / W, T_ch / P)`. This stage sits between batcher and +//! inserter pool so the two overlap, `W` wide to match the bridge — that is +//! how many requests the shadow answers at once. +//! +//! Costs one extra resident batch per inserter, which is what a queue deep +//! enough to keep every inserter fed takes. +//! +//! A batch with no oracle column crosses unchanged, without touching the +//! bridge. + +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use clickhouse_c::Allocator; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use crate::ch::{EmitterError, backoff_step}; +use crate::config::ResolvedConfig; +use crate::emit::ch_emitter::{ColumnBuf, EmitterStats, RetryConfig, literal_column}; +use crate::emit::pipeline::Fatal; +use crate::emit::pipeline::batcher::InsertBatch; +use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; + +/// A sealed batch past the oracle. `local` is by ordinal, `Some` where the +/// daemon built an oracle-routed column itself and left it out of the +/// request; the inserter splices from there instead of from `resolved` +pub(crate) struct ResolvedBatch { + pub batch: InsertBatch, + pub local: Vec>, + pub resolved: Option, +} + +/// `None` oracle resolves nothing, which a pipeline with no oracle column +/// and a pipeline that has one but no shadow both look like; the second +/// fails at its first oracle batch +#[derive(Clone)] +pub(crate) struct ResolverOptions { + pub oracle: Option>, + /// Boot budget for oracle transport retries, re-read from `config_rx` + pub retry: RetryConfig, + pub stats: Arc, + pub fatal: Fatal, + pub config_rx: Option>>, +} + +/// Spawn `n` resolvers over the shared batch queue. Drops the sender it was +/// handed, so the inserters' queue closes once every resolver exits +pub(crate) fn spawn_pool( + n: usize, + rx: async_channel::Receiver, + tx: async_channel::Sender, + opts: ResolverOptions, +) -> Vec> { + (0..n.max(1)) + .map(|_| tokio::spawn(run(rx.clone(), tx.clone(), opts.clone()))) + .collect() +} + +async fn run( + rx: async_channel::Receiver, + tx: async_channel::Sender, + opts: ResolverOptions, +) { + let ResolverOptions { + oracle, + mut retry, + stats, + fatal, + config_rx, + } = opts; + let alloc = Allocator::global(&mimalloc::MiMalloc); + while let Ok(batch) = rx.recv().await { + if let Some(rx) = config_rx.as_ref() { + retry.max_attempts = rx.borrow().retry_max_attempts; + } + let local = local_columns(&batch, &stats); + let started = std::time::Instant::now(); + let resolved = + match resolve_oracle_with_retry(&oracle, alloc, &batch, &local, &retry, &stats).await { + Ok(v) => v, + Err(e) => { + // Seq stays unacknowledged, so a restart replays it + fatal.set(format!("oracle resolve: {e}")); + return; + } + }; + stats + .oracle_resolve_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + if tx + .send(ResolvedBatch { + batch, + local, + resolved, + }) + .await + .is_err() + { + return; + } + } +} + +/// Oracle columns the daemon answers itself, by ordinal. Empty when the +/// batch has no oracle column at all, which is the common shape +fn local_columns(batch: &InsertBatch, stats: &EmitterStats) -> Vec> { + if !batch + .buffers + .iter() + .any(|b| matches!(b, ColumnBuf::Oracle(_))) + { + return Vec::new(); + } + let taken: Vec> = batch + .buffers + .iter() + .enumerate() + .map(|(i, buf)| match buf { + ColumnBuf::Oracle(o) => { + literal_column(o, &batch.meta.columns[i].type_repr, batch.n_rows) + } + _ => None, + }) + .collect(); + let n = taken.iter().filter(|c| c.is_some()).count() as u64; + if n > 0 { + stats.oracle_local_columns.fetch_add(n, Ordering::Relaxed); + } + taken +} + +/// Retry transport failures only +async fn resolve_oracle_with_retry( + oracle: &Option>, + alloc: Allocator, + batch: &InsertBatch, + local: &[Option], + retry: &RetryConfig, + stats: &EmitterStats, +) -> Result, EmitterError> { + let mut attempt = 0u32; + let mut backoff = retry.initial_backoff; + loop { + match resolve_oracle(oracle, alloc, batch, local).await { + Ok(v) => return Ok(v), + Err(e) if e.retryable() && attempt < retry.max_attempts => { + stats.retries_attempted.fetch_add(1, Ordering::Relaxed); + attempt += 1; + backoff_step(&mut backoff, retry.max_backoff).await; + } + Err(e) => return Err(EmitterError::Type(e.to_string())), + } + } +} + +async fn resolve_oracle( + oracle: &Option>, + alloc: Allocator, + batch: &InsertBatch, + local: &[Option], +) -> Result, OracleError> { + let columns: Vec> = batch + .buffers + .iter() + .enumerate() + .filter_map(|(i, buf)| match buf { + ColumnBuf::Oracle(o) if local.get(i).is_none_or(Option::is_none) => { + Some(OracleRequestColumn { + ordinal: i as u32, + name: &batch.meta.columns[i].name, + target_type: &batch.meta.columns[i].type_repr, + buf: o, + }) + } + _ => None, + }) + .collect(); + if columns.is_empty() { + return Ok(None); + } + let oracle = oracle.as_ref().ok_or_else(|| { + OracleError::Absent(format!( + "{} needs the shadow oracle, which this pipeline has none of", + batch.meta.table_key + )) + })?; + oracle + .encode_batch(&columns, batch.n_rows, alloc) + .await + .map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decode::heap_decoder::{ + ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, + }; + use crate::emit::pipeline::batcher::{BatcherConfig, BatcherMsg, RoutedRow}; + use crate::emit::route::RouteSnapshot; + use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; + use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; + use std::time::Duration; + use tokio::sync::{mpsc, oneshot}; + use walrus::pg::walparser::RelFileNode; + + /// An oracle-routed column the local matrix does not cover, rendered by + /// the daemon: the shape a PostGIS 2-D point takes + const GEOM_OID: u32 = 999_999; + + fn wkt_row(seq: u64, text: Option<&str>) -> RoutedRow { + let rel = Arc::new(RelDescriptor { + rfn: RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node: 16385, + }, + oid: 16385, + toast_oid: 0, + namespace_oid: 2200, + rel_name: RelName::new("public", "g"), + kind: 'r', + persistence: 'p', + replident: ReplIdent::Default { pk_attnums: None }, + attributes: vec![RelAttr { + attnum: 1, + name: "geom".into(), + type_oid: GEOM_OID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "geometry".into(), + type_byval: false, + type_len: -1, + type_align: 'i', + type_storage: 'x', + missing_text: None, + }], + }); + let route = RouteSnapshot::freeze( + Arc::new(TableMapping { + target: TableTarget::new("default", "g"), + columns: vec![ColumnMapping { + src_attnum: 1, + target_name: "geom".into(), + target_type: "Nullable(String)".into(), + }], + }), + Arc::default(), + Default::default(), + ); + RoutedRow { + seq, + rel, + route, + committed: CommittedTuple { + decoded: DecodedHeap { + rfn: RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node: 16385, + }, + xid: 7, + source_lsn: 0x1000 + seq, + op: HeapOp::Insert, + new: Some(DecodedTuple { + columns: vec![text.map(|t| ColumnValue::Text(t.into()))], + partial: false, + }), + old: None, + }, + commit_ts: 0, + commit_lsn: (seq + 1) * 100, + }, + value_permit: None, + } + } + + /// One batch of the rows, sealed through the real batcher: `InsertBatch` + /// is only reachable that way, and its cell tags are what the encoder + /// chose rather than what a test asserted + async fn seal(rows: Vec) -> InsertBatch { + let (msg_tx, msg_rx) = mpsc::channel(64); + let (batches_tx, batches_rx) = async_channel::bounded(64); + let fatal = Fatal::new(); + let handle = crate::emit::pipeline::batcher::spawn( + msg_rx, + batches_tx, + BatcherConfig { + row_budget: 1_000, + byte_budget: 1 << 30, + flush_timeout: Duration::from_secs(3600), + }, + Allocator::stdlib(), + fatal.clone(), + Arc::new(EmitterStats::default()), + None, + ); + for r in rows { + msg_tx.send(BatcherMsg::Row(r)).await.expect("send row"); + } + let (reply_tx, reply_rx) = oneshot::channel(); + msg_tx + .send(BatcherMsg::FlushAll(reply_tx)) + .await + .expect("send flush"); + reply_rx.await.expect("flush ack"); + let batch = batches_rx.recv().await.expect("one batch"); + drop(msg_tx); + handle.await.expect("batcher task"); + assert!(fatal.message().is_none()); + batch + } + + /// Cells the daemon rendered against a `String` target never reach the + /// bridge: the same batch resolves with no oracle at all, and fails + /// without the local path + #[tokio::test] + async fn rendered_cells_against_a_string_target_stay_local() { + let batch = seal(vec![ + wkt_row(0, Some("POINT(1 2)")), + wkt_row(0, None), + wkt_row(0, Some("POINT(3 4)")), + ]) + .await; + assert!( + matches!(batch.buffers[0], ColumnBuf::Oracle(_)), + "premise: an uncovered source type routes to the oracle", + ); + + let stats = EmitterStats::default(); + let local = local_columns(&batch, &stats); + assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 1); + let Some(ColumnBuf::NullableString { + offsets, + data, + null_map, + }) = local[0].as_ref() + else { + panic!("column not built locally: {:?}", local[0]); + }; + assert_eq!(data, b"POINT(1 2)POINT(3 4)"); + assert_eq!(offsets, &[10, 10, 20], "absent cell holds its offset"); + assert_eq!(null_map, &[0, 1, 0]); + + let alloc = Allocator::stdlib(); + assert!( + resolve_oracle(&None, alloc, &batch, &local) + .await + .expect("no request to make") + .is_none(), + ); + // Same batch without the local build has to ask someone + assert!(matches!( + resolve_oracle(&None, alloc, &batch, &[]).await, + Err(OracleError::Absent(_)), + )); + } + + /// A cell PG must convert keeps the whole column in the request, even + /// when its neighbours are rendered + #[tokio::test] + async fn one_unrendered_cell_keeps_the_column_remote() { + let mut rows = vec![wkt_row(0, Some("POINT(1 2)"))]; + let mut raw = wkt_row(0, None); + raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + type_oid: GEOM_OID, + raw: vec![1, 2, 3], + }); + rows.push(raw); + let batch = seal(rows).await; + + let stats = EmitterStats::default(); + let local = local_columns(&batch, &stats); + assert!(local[0].is_none()); + assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 0); + } +} diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 9348da00..1b1d4a85 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -23,6 +23,7 @@ use crate::emit::ch_emitter::{EmitterConfig, EmitterStats}; use crate::emit::pipeline::ack::{self, AckHandle}; use crate::emit::pipeline::batcher::{self, BatcherConfig, BatcherMsg, InsertBatch}; use crate::emit::pipeline::inserter; +use crate::emit::pipeline::resolver::{self, ResolvedBatch}; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; use crate::pos::{EmitterAck, Monotone}; use ahash::{HashMap, HashMapExt}; @@ -31,6 +32,7 @@ use ahash::{HashMap, HashMapExt}; pub struct TailParts { collector: JoinHandle<()>, batcher: JoinHandle<()>, + resolvers: Vec>, inserters: Vec>, } @@ -40,6 +42,9 @@ impl TailParts { /// channel close and this hangs. pub async fn join(self) { let _ = self.batcher.await; + for h in self.resolvers { + let _ = h.await; + } for h in self.inserters { let _ = h.await; } @@ -146,6 +151,7 @@ pub fn spawn_null( TailParts { collector, batcher, + resolvers: Vec::new(), inserters: Vec::new(), }, ) @@ -171,21 +177,37 @@ pub async fn spawn_with_config( // enqueued before it let (msg_tx, msg_rx) = mpsc::channel::(256); let (batches_tx, batches_rx) = async_channel::bounded::((n * 2).max(4)); + // One resolved batch per inserter is what keeps every one of them fed + let (resolved_tx, resolved_rx) = async_channel::bounded::(n); let inserters = inserter::spawn_pool( n, emitter, - batches_rx, + resolved_rx, ack.clone(), stats.clone(), fatal.clone(), inserter::PoolOptions { config_rx: config_rx.clone(), - oracle, }, ) .await?; + // As wide as the bridge: a narrower pool leaves shadow workers idle, a + // wider one only queues on their sockets + let resolvers = resolver::spawn_pool( + oracle.as_ref().map_or(1, |o| o.concurrency()), + batches_rx, + resolved_tx, + resolver::ResolverOptions { + oracle, + retry: emitter.retry.clone(), + stats: stats.clone(), + fatal: fatal.clone(), + config_rx: config_rx.clone(), + }, + ); + // Boot fallback for the batcher; the live path re-reads from `config_rx`. let flush_timeout = if emitter.flush_timeout.is_zero() { DEFAULT_PIPELINE_FLUSH @@ -212,6 +234,7 @@ pub async fn spawn_with_config( TailParts { collector, batcher, + resolvers, inserters, }, )) diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index bbb517b6..57cb1c9a 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -10,7 +10,7 @@ use std::io; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -27,11 +27,17 @@ pub const PROJECTION_VERSION: u32 = 1; /// Match `WS_MAX_REQUEST_BYTES` pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024; +/// `len: u32be` then `op: u8`. Request builders reserve this much leading +/// room so the bridge patches the prefix in place and one `write_all` ships +/// the frame, rather than reallocating and copying the whole payload +pub const FRAME_PREFIX_BYTES: usize = 5; /// Whole-catalog `pg_type` text output is the largest response in practice const MAX_RESPONSE_BYTES: usize = 256 * 1024 * 1024; /// Matches `WS_MAX_SCAN_OIDS`. A longer list is the caller's to chunk, since /// only the caller knows whether the chunks share a replay position pub const MAX_SCAN_OIDS: usize = 65536; +/// Matches `WS_MAX_WORKERS`, the ceiling on `walshadow.bridge_workers` +pub const MAX_BRIDGE_WORKERS: usize = 8; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] @@ -50,6 +56,13 @@ impl Op { fn slot(self) -> usize { self as usize - 1 } + + /// Catalog reads pin to worker 0. `SCAN` answers off a replay position + /// it reports back, and `HELLO` establishes the identity every other + /// worker is then checked against, so neither may drift between sockets + fn pinned(self) -> bool { + matches!(self, Op::Hello | Op::Scan) + } } /// Catalogs the overlay scan covers. Ids are wire values; never renumber @@ -170,10 +183,20 @@ crate::atomic_stats! { /// reads answer these off SQL instead; overlay reads fail pub scan_replay_moved, pub native_bytes, + /// Bridge sockets the pool holds, one per shadow-side worker + pub pool_size, /// Per-op, indexed by [`OP_LABELS`] pub requests: [AtomicU64; OP_COUNT], pub errors: [AtomicU64; OP_COUNT], pub request_nanos: [AtomicU64; OP_COUNT], + /// Queued behind another caller's request on the one socket. Against + /// `service_nanos` this is what says whether the worker is the + /// limiter or the funnel in front of it is + pub lock_wait_nanos: [AtomicU64; OP_COUNT], + /// Wire time with the socket held: worker conversion plus transfer + pub service_nanos: [AtomicU64; OP_COUNT], + pub request_bytes: [AtomicU64; OP_COUNT], + pub response_bytes: [AtomicU64; OP_COUNT], } } @@ -203,11 +226,23 @@ impl BridgeStats { } } +/// One shadow-side worker's socket. A worker serves one request at a time, +/// so the mutex is the worker, not an artefact of sharing #[derive(Debug)] -pub struct Bridge { +struct Slot { path: PathBuf, conn: Mutex>, - /// Set by the first successful `HELLO`; later dials must match it +} + +#[derive(Debug)] +pub struct Bridge { + /// Slot 0 is `walshadow.socket_path`, slot `i` is `socket_path.i`. + /// Stateless ops round-robin; [`Op::pinned`] ops stay on slot 0 + slots: Vec, + next: AtomicUsize, + /// Set by the first successful `HELLO`; later dials, on any slot, must + /// match it. A worker that came back a different build means a mixed + /// install and must fail closed info: OnceLock, pub stats: Arc, } @@ -217,19 +252,54 @@ impl Bridge { /// mismatch is refused rather than negotiated: the daemon would misparse /// the projections pub async fn connect(path: impl AsRef) -> Result { + Self::connect_pooled(path, 1).await + } + + /// One socket per shadow-side worker, matching + /// `walshadow.bridge_workers`. Every slot must answer: a pool short of + /// the configured width is a half-started shadow, and silently running + /// narrower would hide it + pub async fn connect_pooled( + path: impl AsRef, + workers: usize, + ) -> Result { + let base = path.as_ref(); + let slots: Vec = (0..workers.clamp(1, MAX_BRIDGE_WORKERS)) + .map(|i| Slot { + path: if i == 0 { + base.to_owned() + } else { + PathBuf::from(format!("{}.{i}", base.display())) + }, + conn: Mutex::new(None), + }) + .collect(); let bridge = Self { - path: path.as_ref().to_owned(), - conn: Mutex::new(None), + slots, + next: AtomicUsize::new(0), info: OnceLock::new(), stats: Arc::new(BridgeStats::default()), }; - let stream = bridge.dial().await?; - *bridge.conn.lock().await = Some(stream); + // Slot 0 first, so its HELLO is the identity the rest are checked + // against rather than whichever worker happened to answer first + for slot in &bridge.slots { + let stream = bridge.dial(slot).await?; + *slot.conn.lock().await = Some(stream); + } + bridge + .stats + .pool_size + .store(bridge.slots.len() as u64, Ordering::Relaxed); Ok(bridge) } + /// `walshadow.socket_path`, ie slot 0 pub fn path(&self) -> &Path { - &self.path + &self.slots[0].path + } + + pub fn pool_size(&self) -> usize { + self.slots.len() } /// `None` before the first successful `HELLO`, which [`connect`](Self::connect) @@ -244,13 +314,14 @@ impl Bridge { /// `pg_last_wal_replay_lsn` by shared-memory read, one round trip pub async fn replay_lsn(&self) -> Result { - let body = self.call(Op::ReplayLsn, &[]).await?; + let body = self.call(Op::ReplayLsn, request_frame(0)).await?; Cursor::at(&body, 1).u64() } - /// Return response remainder as one locally framed Native block - pub async fn encode_native(&self, payload: &[u8]) -> Result { - let frame = self.call(Op::EncodeNative, payload).await?; + /// Return response remainder as one locally framed Native block. + /// `frame` carries [`FRAME_PREFIX_BYTES`] of unwritten leading room + pub async fn encode_native(&self, frame: Vec) -> Result { + let frame = self.call(Op::EncodeNative, frame).await?; self.stats .native_bytes .fetch_add((frame.len() - 1) as u64, Ordering::Relaxed); @@ -270,15 +341,15 @@ impl Bridge { top_xid: u32, oids: &[u32], ) -> Result { - let mut payload = Vec::with_capacity(9 + oids.len() * 4); - payload.push(cat as u8); - payload.extend_from_slice(&top_xid.to_be_bytes()); - payload.extend_from_slice(&(oids.len() as u32).to_be_bytes()); + let mut frame = request_frame(9 + oids.len() * 4); + frame.push(cat as u8); + frame.extend_from_slice(&top_xid.to_be_bytes()); + frame.extend_from_slice(&(oids.len() as u32).to_be_bytes()); for oid in oids { - payload.extend_from_slice(&oid.to_be_bytes()); + frame.extend_from_slice(&oid.to_be_bytes()); } - let body = self.call(Op::Scan, &payload).await?; + let body = self.call(Op::Scan, frame).await?; let mut c = Cursor::at(&body, 1); let replay_lsn_start = c.u64()?; let replay_lsn_end = c.u64()?; @@ -367,10 +438,13 @@ impl Bridge { /// Fresh socket plus `HELLO`. Takes no connection lock, so /// [`call`](Self::call) may hold one across it - async fn dial(&self) -> Result { - let mut stream = UnixStream::connect(&self.path).await?; + async fn dial(&self, slot: &Slot) -> Result { + let mut stream = UnixStream::connect(&slot.path).await?; + widen_sockbufs(&stream); let started = Instant::now(); - let res = round_trip(&mut stream, Op::Hello, &[]).await; + let mut hello = request_frame(0); + patch_frame(&mut hello, Op::Hello); + let res = round_trip(&mut stream, &hello).await; self.record(Op::Hello, started, &res); let body = res?; @@ -400,11 +474,13 @@ impl Bridge { Ok(stream) } - async fn call(&self, op: Op, payload: &[u8]) -> Result, BridgeError> { + /// `frame` carries [`FRAME_PREFIX_BYTES`] of unwritten leading room, + /// patched here rather than copied into a second buffer + async fn call(&self, op: Op, mut frame: Vec) -> Result, BridgeError> { let started = Instant::now(); // Refuse before the socket sees it: the worker answers a frame this // size by closing, and a healthy connection must not pay for that - let len = payload.len() + 1; + let len = frame.len() - FRAME_PREFIX_BYTES + 1; if len > MAX_REQUEST_BYTES { let res = Err(BridgeError::RequestTooLarge { len, @@ -413,9 +489,18 @@ impl Bridge { self.record(op, started, &res); return res; } - let mut guard = self.conn.lock().await; + patch_frame(&mut frame, op); + let stat = op.slot(); + self.stats.request_bytes[stat].fetch_add(frame.len() as u64, Ordering::Relaxed); + let slot = self.pick(op); + let mut guard = slot.conn.lock().await; + let held = Instant::now(); + self.stats.lock_wait_nanos[stat].fetch_add( + held.duration_since(started).as_nanos() as u64, + Ordering::Relaxed, + ); let mut res = match guard.as_mut() { - Some(stream) => round_trip(stream, op, payload).await, + Some(stream) => round_trip(stream, &frame).await, None => Err(BridgeError::Io(io::Error::new( io::ErrorKind::NotConnected, "bridge disconnected", @@ -426,9 +511,9 @@ impl Bridge { if is_transport_error(&res) { *guard = None; self.stats.reconnects.fetch_add(1, Ordering::Relaxed); - match self.dial().await { + match self.dial(slot).await { Ok(mut stream) => { - res = round_trip(&mut stream, op, payload).await; + res = round_trip(&mut stream, &frame).await; if !is_transport_error(&res) { *guard = Some(stream); } @@ -437,10 +522,26 @@ impl Bridge { } } drop(guard); + self.stats.service_nanos[stat] + .fetch_add(held.elapsed().as_nanos() as u64, Ordering::Relaxed); + if let Ok(body) = &res { + self.stats.response_bytes[stat].fetch_add(body.len() as u64, Ordering::Relaxed); + } self.record(op, started, &res); res } + /// Round-robin over the pool, except for ops pinned to worker 0. + /// `ENCODE_NATIVE` is stateless and read-only, so any worker answers any + /// request + fn pick(&self, op: Op) -> &Slot { + if op.pinned() || self.slots.len() == 1 { + return &self.slots[0]; + } + let i = self.next.fetch_add(1, Ordering::Relaxed) % self.slots.len(); + &self.slots[i] + } + fn record(&self, op: Op, started: Instant, res: &Result, BridgeError>) { let slot = op.slot(); self.stats.requests[slot].fetch_add(1, Ordering::Relaxed); @@ -463,19 +564,50 @@ fn is_transport_error(res: &Result, BridgeError>) -> bool { matches!(res, Err(e) if e.is_transport()) } -async fn round_trip( - stream: &mut UnixStream, - op: Op, - payload: &[u8], -) -> Result, BridgeError> { - let len = payload.len() + 1; - // One write: a peer that dribbles a partial frame is what the worker's - // io_timeout_ms exists to bound, and the daemon must not be that peer - let mut frame = Vec::with_capacity(4 + len); - frame.extend_from_slice(&(len as u32).to_be_bytes()); - frame.push(op as u8); - frame.extend_from_slice(payload); - stream.write_all(&frame).await?; +/// Matches `WS_SOCKBUF_BYTES` in `pgext/worker.c`. A multi-megabyte frame +/// against default socket buffers costs hundreds of `EAGAIN` round trips +/// on each side, so both ends ask for a wide window +const SOCKBUF_BYTES: libc::c_int = 4 * 1024 * 1024; + +/// Advisory: a kernel that refuses leaves the default, which only costs +/// more wakeups. The worker widens its own end on accept +fn widen_sockbufs(stream: &UnixStream) { + use std::os::fd::AsRawFd; + let fd = stream.as_raw_fd(); + let want = SOCKBUF_BYTES; + for opt in [libc::SO_RCVBUF, libc::SO_SNDBUF] { + // SAFETY: `fd` is owned by `stream` and outlives the call; `want` is + // a live `c_int` of the length passed + unsafe { + libc::setsockopt( + fd, + libc::SOL_SOCKET, + opt, + (&raw const want).cast(), + std::mem::size_of_val(&want) as libc::socklen_t, + ); + } + } +} + +/// Leading room for the length + opcode prefix, unwritten until the bridge +/// knows both +pub fn request_frame(payload_capacity: usize) -> Vec { + let mut v = Vec::with_capacity(FRAME_PREFIX_BYTES + payload_capacity); + v.resize(FRAME_PREFIX_BYTES, 0); + v +} + +fn patch_frame(frame: &mut [u8], op: Op) { + let len = (frame.len() - FRAME_PREFIX_BYTES + 1) as u32; + frame[..4].copy_from_slice(&len.to_be_bytes()); + frame[4] = op as u8; +} + +/// One write: a peer that dribbles a partial frame is what the worker's +/// io_timeout_ms exists to bound, and the daemon must not be that peer +async fn round_trip(stream: &mut UnixStream, frame: &[u8]) -> Result, BridgeError> { + stream.write_all(frame).await?; stream.flush().await?; let mut hdr = [0u8; 4]; @@ -499,9 +631,13 @@ async fn round_trip( /// Connect with a wall-clock budget while shadow reaches consistency. /// Matches catalog's /// [`with_transient_retry`](crate::catalog::shadow_catalog::with_transient_retry) shape -pub async fn connect_with_budget(path: &Path, budget: Duration) -> Result { +pub async fn connect_with_budget( + path: &Path, + workers: usize, + budget: Duration, +) -> Result { let deadline = tokio::time::Instant::now() + budget; - (|| Bridge::connect(path)) + (|| Bridge::connect_pooled(path, workers)) .retry( ExponentialBuilder::default() .with_min_delay(Duration::from_millis(100)) @@ -948,7 +1084,9 @@ mod tests { ]); let bridge = Bridge::connect(&path).await.unwrap(); - let out = bridge.encode_native(&[0u8; 8]).await.unwrap(); + let mut req = request_frame(8); + req.extend_from_slice(&[0u8; 8]); + let out = bridge.encode_native(req).await.unwrap(); assert_eq!(out.bytes(), &native[..]); assert_eq!( bridge.stats.native_bytes.load(Ordering::Relaxed), @@ -1042,8 +1180,9 @@ mod tests { let (_tmp, path) = spawn_worker(vec![Some(hello_body(PROTO_VERSION, PROJECTION_VERSION))]); let bridge = Bridge::connect(&path).await.unwrap(); - let huge = vec![0u8; MAX_REQUEST_BYTES]; - let err = bridge.encode_native(&huge).await.unwrap_err(); + let mut huge = request_frame(MAX_REQUEST_BYTES); + huge.resize(FRAME_PREFIX_BYTES + MAX_REQUEST_BYTES, 0); + let err = bridge.encode_native(huge).await.unwrap_err(); assert!( matches!(err, BridgeError::RequestTooLarge { .. }), "got {err:?}" diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 1a8ad34b..758b04e7 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -78,6 +78,16 @@ pub struct MetricsSnapshot { pub bootstrap_deferred_bytes: u64, /// Encoded bytes in the bootstrap TOAST-deferred spool file pub bootstrap_deferred_spool_bytes: u64, + /// Bootstrap pump stage attribution. Live while a greenfield bootstrap + /// runs, then frozen at its final values for the rest of the session + pub bootstrap_bytes_tapped: u64, + pub bootstrap_pages_walked: u64, + pub bootstrap_tuples_emitted: u64, + pub bootstrap_files_walked: u64, + pub bootstrap_files_skipped_unmapped: u64, + pub bootstrap_decode_seconds: f64, + pub bootstrap_tap_seconds: f64, + pub bootstrap_channel_block_seconds: f64, pub spill_evictions_total: u64, pub xacts_committed_total: u64, pub xacts_aborted_total: u64, @@ -150,8 +160,12 @@ pub struct MetricsSnapshot { pub insertbatch_rows_in_total: u64, pub insertbatch_batches_out_total: u64, pub inserter_batches_in_total: u64, + pub inserter_ch_seconds_total: f64, + pub inserter_encode_seconds_total: f64, + pub oracle_resolve_seconds_total: f64, pub process_cpu_seconds_total: f64, pub process_resident_memory_bytes: u64, + pub oracle_local_columns_total: u64, pub oracle_blocks_total: u64, pub oracle_rows_total: u64, pub oracle_cells_total: u64, @@ -165,6 +179,12 @@ pub struct MetricsSnapshot { pub bridge_requests_by_op: [u64; 4], pub bridge_errors_by_op: [u64; 4], pub bridge_request_nanos_by_op: [u64; 4], + /// Queued behind another caller on the single bridge socket + pub bridge_lock_wait_nanos_by_op: [u64; 4], + /// Wire time with the socket held + pub bridge_service_nanos_by_op: [u64; 4], + pub bridge_request_bytes_by_op: [u64; 4], + pub bridge_response_bytes_by_op: [u64; 4], pub bridge_reconnects_total: u64, pub bridge_scan_rows_total: u64, pub bridge_scan_subtrans_mismatch_total: u64, @@ -573,6 +593,36 @@ pub fn render(snap: &MetricsSnapshot) -> String { "gauge", snap.bootstrap_deferred_spool_bytes, ), + ( + "walshadow_bootstrap_bytes_tapped_total", + "Backup body bytes the bootstrap pump handed to the page-walk sink.", + "counter", + snap.bootstrap_bytes_tapped, + ), + ( + "walshadow_bootstrap_pages_walked_total", + "8 KiB heap pages the bootstrap walk framed.", + "counter", + snap.bootstrap_pages_walked, + ), + ( + "walshadow_bootstrap_tuples_emitted_total", + "Live tuples the bootstrap walk decoded off backup pages.", + "counter", + snap.bootstrap_tuples_emitted, + ), + ( + "walshadow_bootstrap_files_walked_total", + "User-heap segments the bootstrap walk decoded.", + "counter", + snap.bootstrap_files_walked, + ), + ( + "walshadow_bootstrap_files_skipped_unmapped_total", + "User-heap segments declined at begin because no mapped relation owns them; their bytes drain unread.", + "counter", + snap.bootstrap_files_skipped_unmapped, + ), ( "walshadow_spill_evictions_total", "Total evictions in→spill since daemon start.", @@ -784,6 +834,12 @@ pub fn render(snap: &MetricsSnapshot) -> String { "gauge", snap.process_resident_memory_bytes, ), + ( + "walshadow_oracle_local_columns_total", + "Oracle-routed columns the daemon built itself: already-rendered cells against a String target, which PG would hand straight back.", + "counter", + snap.oracle_local_columns_total, + ), ( "walshadow_oracle_blocks_total", "Partial Native blocks the walshadow extension returned and the daemon validated.", @@ -1278,12 +1334,47 @@ pub fn render(snap: &MetricsSnapshot) -> String { for (op, v) in OP_LABELS.iter().zip(snap.bridge_errors_by_op) { writeln!(s, "{name}{{op=\"{op}\"}} {v}").unwrap(); } - let name = "walshadow_bridge_request_seconds_total"; - writeln!(s, "# HELP {name} Wall time spent in bridge round trips.").unwrap(); - writeln!(s, "# TYPE {name} counter").unwrap(); - for (op, v) in OP_LABELS.iter().zip(snap.bridge_request_nanos_by_op) { - let secs = v as f64 / 1e9; - writeln!(s, "{name}{{op=\"{op}\"}} {secs}").unwrap(); + for (name, help, nanos) in [ + ( + "walshadow_bridge_request_seconds_total", + "Wall time spent in bridge round trips.", + snap.bridge_request_nanos_by_op, + ), + ( + "walshadow_bridge_lock_wait_seconds_total", + "Wall time bridge callers spent queued for the socket. Against bridge_service_seconds this says whether the worker or the funnel in front of it is the limiter.", + snap.bridge_lock_wait_nanos_by_op, + ), + ( + "walshadow_bridge_service_seconds_total", + "Wall time on the wire with the bridge socket held: worker conversion plus transfer.", + snap.bridge_service_nanos_by_op, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + for (op, v) in OP_LABELS.iter().zip(nanos) { + let secs = v as f64 / 1e9; + writeln!(s, "{name}{{op=\"{op}\"}} {secs}").unwrap(); + } + } + for (name, help, bytes) in [ + ( + "walshadow_bridge_request_bytes_total", + "Request frame bytes written to the bridge socket.", + snap.bridge_request_bytes_by_op, + ), + ( + "walshadow_bridge_response_bytes_total", + "Response frame bytes read back off the bridge socket.", + snap.bridge_response_bytes_by_op, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + for (op, v) in OP_LABELS.iter().zip(bytes) { + writeln!(s, "{name}{{op=\"{op}\"}} {v}").unwrap(); + } } } @@ -1339,6 +1430,43 @@ pub fn render(snap: &MetricsSnapshot) -> String { writeln!(s, "# TYPE {name} counter").unwrap(); writeln!(s, "{name} {:.3}", snap.desc_capture_seconds_total).unwrap(); + for (name, help, secs) in [ + ( + "walshadow_bootstrap_decode_seconds_total", + "Cumulative CPU inside the bootstrap page walk, tuple decode included.", + snap.bootstrap_decode_seconds, + ), + ( + "walshadow_bootstrap_tap_seconds_total", + "Cumulative time bootstrap tap readers spent inside the sink: page framing, decode, channel send. Against bootstrap_decode_seconds this is the tap's own overhead.", + snap.bootstrap_tap_seconds, + ), + ( + "walshadow_bootstrap_channel_block_seconds_total", + "Cumulative time the bootstrap walk spent waiting for a free tuple-channel slot, i.e. emitter drain time seen by the walk.", + snap.bootstrap_channel_block_seconds, + ), + ( + "walshadow_inserter_ch_seconds_total", + "Cumulative inserter time inside the ClickHouse INSERT round trip. Against inserter_pool_size x uptime this is CH-side utilization.", + snap.inserter_ch_seconds_total, + ), + ( + "walshadow_inserter_encode_seconds_total", + "Cumulative inserter time rebuilding the Native block over a batch's owned slabs.", + snap.inserter_encode_seconds_total, + ), + ( + "walshadow_oracle_resolve_seconds_total", + "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", + snap.oracle_resolve_seconds_total, + ), + ] { + writeln!(s, "# HELP {name} {help}").unwrap(); + writeln!(s, "# TYPE {name} counter").unwrap(); + writeln!(s, "{name} {secs:.3}").unwrap(); + } + // Process CPU as a float counter (seconds); rate() ≈ cores in use. let name = "walshadow_process_cpu_seconds_total"; writeln!( @@ -1471,6 +1599,37 @@ mod tests { assert!(body.contains("walshadow_route_snapshots_total{result=\"unmapped\"} 8")); } + /// Bootstrap stage attribution reaches `/metrics`. `BootstrapOutcome` + /// counters used to stop at a log line, which is what made a slow + /// initial load unattributable + #[test] + fn render_exposes_bootstrap_stage_attribution() { + let snap = MetricsSnapshot { + bootstrap_bytes_tapped: 1 << 30, + bootstrap_pages_walked: 131_072, + bootstrap_files_skipped_unmapped: 297, + bootstrap_decode_seconds: 12.5, + bootstrap_channel_block_seconds: 3.25, + bootstrap_tap_seconds: 0.0, + inserter_ch_seconds_total: 41.5, + oracle_resolve_seconds_total: 8.25, + ..MetricsSnapshot::default() + }; + let body = render(&snap); + for want in [ + "walshadow_bootstrap_bytes_tapped_total 1073741824", + "walshadow_bootstrap_pages_walked_total 131072", + "walshadow_bootstrap_files_skipped_unmapped_total 297", + "walshadow_bootstrap_decode_seconds_total 12.500", + "walshadow_bootstrap_channel_block_seconds_total 3.250", + "walshadow_bootstrap_tap_seconds_total 0.000", + "walshadow_inserter_ch_seconds_total 41.500", + "walshadow_oracle_resolve_seconds_total 8.250", + ] { + assert!(body.contains(want), "missing {want}"); + } + } + /// Prometheus rejects a family declared twice; `descriptor_ambiguous_total` /// was emitted bare and labelled with contradicting HELP text #[test] diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 298935f3..051f6ab3 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -6,7 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use clickhouse_c::{Allocator, Block, BlockOpts, BlockReader, Column, SliceIo}; use crate::decode::heap_decoder::ColumnValue; -use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES}; +use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES, request_frame}; use crate::schema::RelAttr; /// Cell tags, matching `WS_CELL_*` in `pgext/walshadow.h` @@ -71,6 +71,7 @@ pub struct OracleColumnBuf { pub source_typmod: i32, cells: Vec, wire_bytes: usize, + literal_only: bool, } impl OracleColumnBuf { @@ -80,14 +81,23 @@ impl OracleColumnBuf { source_typmod, cells: Vec::new(), wire_bytes: 0, + literal_only: true, } } pub fn push(&mut self, cell: OracleCell) { + self.literal_only &= matches!(cell, OracleCell::Literal(_) | OracleCell::Default); self.wire_bytes += cell.wire_bytes(); self.cells.push(cell); } + /// No cell needs a source type: each is either bytes the daemon already + /// rendered or a default. Against a `String` target such a column would + /// cross to PG only to be handed straight back + pub fn literal_only(&self) -> bool { + self.literal_only + } + pub fn cells(&self) -> &[OracleCell] { &self.cells } @@ -141,6 +151,12 @@ impl Oracle { } } + /// Requests the shadow answers at once, ie the bridge's pool width. One + /// worker serves one request per loop iteration + pub fn concurrency(&self) -> usize { + self.bridge.pool_size() + } + pub async fn encode_batch( &self, columns: &[OracleRequestColumn<'_>], @@ -158,8 +174,11 @@ impl Oracle { ))); } } - let payload = encode_request(columns, n_rows); - let response = match self.bridge.encode_native(&payload).await { + let response = match self + .bridge + .encode_native(encode_request(columns, n_rows)) + .await + { Ok(r) => r, Err(e) => { if matches!(e, BridgeError::Remote(_)) { @@ -201,13 +220,16 @@ pub const ORACLE_BATCH_SEAL_BYTES: usize = 32 << 20; const _: () = assert!(ORACLE_BATCH_SEAL_BYTES <= MAX_REQUEST_BYTES); -/// Encode all column metadata before row-major cells +/// Encode all column metadata before row-major cells, past the bridge's +/// unwritten frame prefix. Seal bytes reach 32 MiB, so building into the +/// frame rather than into a payload the bridge then copies saves one +/// allocation and one memcpy of the whole request fn encode_request(columns: &[OracleRequestColumn<'_>], n_rows: usize) -> Vec { let size: usize = columns .iter() .map(|c| request_column_bytes(c.name, c.target_type) + c.buf.approx_size()) .sum(); - let mut out = Vec::with_capacity(8 + size); + let mut out = request_frame(8 + size); out.extend_from_slice(&(n_rows as u32).to_be_bytes()); out.extend_from_slice(&(columns.len() as u32).to_be_bytes()); for c in columns { @@ -364,6 +386,7 @@ impl OracleStats { #[cfg(test)] mod tests { use super::*; + use crate::ops::bridge::FRAME_PREFIX_BYTES; fn col<'a>( ordinal: u32, @@ -390,7 +413,7 @@ mod tests { let cols = [col(0, "t", "Array(Int32)", &a), col(3, "j", "JSON", &b)]; let out = encode_request(&cols, 2); - let mut c = 0; + let mut c = FRAME_PREFIX_BYTES; let u32_at = |c: &mut usize| { let v = u32::from_be_bytes(out[*c..*c + 4].try_into().unwrap()); *c += 4; @@ -439,7 +462,7 @@ mod tests { buf.push(OracleCell::DiskRaw(vec![1, 2, 3])); let cols = [col(7, "payload", "Nullable(JSON)", &buf)]; assert_eq!( - encode_request(&cols, 2).len() + 1, // opcode the bridge prepends + encode_request(&cols, 2).len() + 1 - FRAME_PREFIX_BYTES, REQUEST_FRAME_BYTES + request_column_bytes("payload", "Nullable(JSON)") + buf.approx_size() diff --git a/tests/bootstrap_direct_e2e.rs b/tests/bootstrap_direct_e2e.rs index ec8f44b4..98634d25 100644 --- a/tests/bootstrap_direct_e2e.rs +++ b/tests/bootstrap_direct_e2e.rs @@ -181,12 +181,23 @@ async fn direct_source_self_hosted_via_replication_protocol() { // DiskLanderSink coverage assert!( - outcome.disk.kept_files > 100, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed) + > 100, "expected >100 catalog files landed, got {}", - outcome.disk.kept_files, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed), ); assert!( - outcome.disk.skipped_denylist > 0, + outcome + .disk + .skipped_denylist + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no denylist files skipped — expected at least one under pg_replslot/ or pg_stat_tmp/", ); @@ -214,15 +225,46 @@ async fn direct_source_self_hosted_via_replication_protocol() { ); // PageWalkSink stats - assert!(outcome.page_walk.files_seen > 0); - assert!(outcome.page_walk.files_walked > 0); - assert!(outcome.page_walk.pages_walked > 0); assert!( - outcome.page_walk.tuples_emitted >= N_ROWS as u64, + outcome + .page_walk + .files_seen + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .files_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .pages_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0 + ); + assert!( + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) + >= N_ROWS as u64, "expected >= {N_ROWS} tuples emitted from page walk, got {}", - outcome.page_walk.tuples_emitted, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), + ); + assert_eq!( + shipped, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) ); - assert_eq!(shipped, outcome.page_walk.tuples_emitted); // CommittedTuple shape let mut int_ids: HashSet = HashSet::new(); diff --git a/tests/bootstrap_object_store_e2e.rs b/tests/bootstrap_object_store_e2e.rs index 516d1ad0..fa23a03f 100644 --- a/tests/bootstrap_object_store_e2e.rs +++ b/tests/bootstrap_object_store_e2e.rs @@ -284,14 +284,25 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // base// + global/. Use a generous lower bound rather than // a tight count: PG version drift changes the exact number. assert!( - outcome.disk.kept_files > 100, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed) + > 100, "expected >100 catalog files landed, got {}; \ tuned for PG initdb expectations — bump or investigate \ a regression", - outcome.disk.kept_files, + outcome + .disk + .kept_files + .load(std::sync::atomic::Ordering::Relaxed), ); assert!( - outcome.disk.skipped_denylist > 0, + outcome + .disk + .skipped_denylist + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no denylist files skipped — expected at least one under pg_replslot/ or pg_stat_tmp/", ); @@ -325,23 +336,49 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // --- PageWalkSink stats --- assert!( - outcome.page_walk.files_seen > 0, + outcome + .page_walk + .files_seen + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no user-heap files observed by the tap", ); assert!( - outcome.page_walk.files_walked > 0, + outcome + .page_walk + .files_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0, "no user-heap files walked (filenode mismatch with CatalogMap?)", ); - assert!(outcome.page_walk.pages_walked > 0, "no heap pages walked",); assert!( - outcome.page_walk.tuples_emitted >= N_ROWS as u64, + outcome + .page_walk + .pages_walked + .load(std::sync::atomic::Ordering::Relaxed) + > 0, + "no heap pages walked", + ); + assert!( + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed) + >= N_ROWS as u64, "expected >= {N_ROWS} tuples emitted from page walk, got {}", - outcome.page_walk.tuples_emitted, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), ); // --- Drain delivered every tuple --- assert_eq!( - shipped, outcome.page_walk.tuples_emitted, + shipped, + outcome + .page_walk + .tuples_emitted + .load(std::sync::atomic::Ordering::Relaxed), "drain count != page walk emit count — channel dropped tuples" ); diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 465622b8..73a82467 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -160,12 +160,12 @@ async fn bootstrap_tail_fans_out_n2() { // Feed all foo rows then all baz rows — contiguous per rfn, as // PageWalkSink emits. Two seqs, each spanning ~8 budget-sized batches. // cap > 2*ROWS_PER_TABLE so the pre-send fits before the drain spawns - let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::(128); + let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::>(128); for id in 0..ROWS_PER_TABLE { - tup_tx.send(tuple(16400, id)).await.unwrap(); + tup_tx.send(vec![tuple(16400, id)]).await.unwrap(); } for id in 0..ROWS_PER_TABLE { - tup_tx.send(tuple(16401, id)).await.unwrap(); + tup_tx.send(vec![tuple(16401, id)]).await.unwrap(); } drop(tup_tx); diff --git a/tests/bridge.rs b/tests/bridge.rs index 1de4b394..8afc8f49 100644 --- a/tests/bridge.rs +++ b/tests/bridge.rs @@ -20,7 +20,7 @@ use walshadow::bridge::{ }; use walshadow::oracle::{Oracle, OracleCell, OracleColumnBuf, OracleRequestColumn}; use walshadow::pg::socket_conninfo; -use walshadow::schema::ReplIdent; +use walshadow::schema::{NUMERICOID, ReplIdent}; use walshadow::shadow::{BridgeConf, Shadow, ShadowConfig}; use walshadow::shadow_catalog::{CatalogError, ShadowCatalog, ShadowCatalogConfig}; @@ -56,6 +56,10 @@ impl Drop for StopOnDrop { } fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { + start_pg_with_workers(tmp, port, 1) +} + +fn start_pg_with_workers(tmp: &tempfile::TempDir, port: u16, workers: usize) -> StopOnDrop { let lib_dir = pgext_dir(); let mut cfg = ShadowConfig::new(tmp.path().join("data"), tmp.path().join("filtered")); cfg.port = port; @@ -63,6 +67,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { cfg.ctl_timeout = Duration::from_secs(60); let mut bridge = BridgeConf::in_dir(&cfg.socket_dir); bridge.library_dir = Some(lib_dir); + bridge.workers = workers; cfg.bridge = Some(bridge); fs::create_dir_all(&cfg.filter_out_dir).unwrap(); fs::create_dir_all(&cfg.socket_dir).unwrap(); @@ -76,7 +81,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> StopOnDrop { async fn dial(sh: &Shadow) -> Bridge { let path = sh.bridge_socket().expect("bridge configured"); - walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())) } @@ -115,7 +120,7 @@ async fn open_catalog(sh: &Shadow, bridge: Arc) -> ShadowCatalog { async fn open_mirror_catalog(sh: &Shadow, sock: &Path) -> (Arc, ShadowCatalog) { spawn_moving_worker(tokio::net::UnixListener::bind(sock).expect("bind stand-in")); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(sock, Duration::from_secs(5)) + walshadow::bridge::connect_with_budget(sock, 1, Duration::from_secs(5)) .await .expect("stand-in bridge"), ); @@ -1027,3 +1032,102 @@ async fn bridge_committed_read_falls_back_when_replay_moves() { "{err:?}" ); } + +/// `walshadow.bridge_workers = 4` registers four workers on +/// `socket_path`, `socket_path.1`, `.2`, `.3`. Pooling them is what stops +/// oracle throughput being one backend's conversion rate. +/// +/// Asserts routing, not rate: four in-flight `ENCODE_NATIVE`s over four +/// sockets each get their own answer back. The pooled-vs-single throughput +/// ratio is hardware-bound (cores cap it) so it is measured by the perf +/// workload, never asserted here, see +/// [`plans/future/perf_regression.md`](../plans/future/perf_regression.md) +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn bridge_worker_pool_serves_concurrent_requests() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + const WORKERS: usize = 4; + let tmp = tempfile::tempdir().unwrap(); + let guard = start_pg_with_workers(&tmp, ports::PG_SHADOW_PORT, WORKERS); + let path = guard.sh.bridge_socket().expect("bridge configured"); + + let pooled = Arc::new( + walshadow::bridge::connect_with_budget(path, WORKERS, Duration::from_secs(30)) + .await + .expect("pooled bridge connect"), + ); + // Same shadow, one socket: a budget below the worker count is honoured + let single = Arc::new( + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(30)) + .await + .expect("single bridge connect"), + ); + assert_eq!(pooled.pool_size(), WORKERS, "one socket per worker"); + assert_eq!(single.pool_size(), 1); + // Every slot dialled its own HELLO; a mismatch would have failed connect + assert!(pooled.info().is_some()); + + // Wide enough that requests are still overlapping when the last one is + // dispatched, which is the state a shared slot would have to serialize + const ROWS: usize = 10_000; + let cells = Arc::new({ + let mut b = OracleColumnBuf::new(NUMERICOID, -1); + for _ in 0..ROWS { + b.push(OracleCell::DiskRaw(numeric_42())); + } + b + }); + + /// One `ENCODE_NATIVE` per task, all in flight at once + async fn race(bridge: &Arc, cells: &Arc, tasks: usize) { + let mut set = Vec::new(); + for _ in 0..tasks { + let bridge = bridge.clone(); + let cells = cells.clone(); + set.push(tokio::spawn(async move { + let columns = [OracleRequestColumn { + ordinal: 0, + name: "c0", + target_type: "String", + buf: &cells, + }]; + Oracle::new(bridge) + .encode_batch( + &columns, + cells.cells().len(), + clickhouse_c::Allocator::stdlib(), + ) + .await + .expect("oracle answers") + .column(0) + .and_then(|c| c.string()) + .expect("string column") + .0 + .len() + })); + } + for h in set { + assert_eq!(h.await.unwrap(), ROWS, "one offset per row"); + } + } + + race(&pooled, &cells, WORKERS).await; + // One socket carrying the same concurrency answers every caller too + race(&single, &cells, WORKERS).await; + + // Pool width is operator-visible + assert_eq!( + pooled + .stats + .pool_size + .load(std::sync::atomic::Ordering::Relaxed), + WORKERS as u64, + ); + // Catalog reads stayed on worker 0 whatever the pool width + pooled + .scan(Catalog::Namespace, 0, &[]) + .await + .expect("scan pinned to slot 0 still answers"); +} diff --git a/tests/common/inproc_harness.rs b/tests/common/inproc_harness.rs index 5721f683..d5ebc3dd 100644 --- a/tests/common/inproc_harness.rs +++ b/tests/common/inproc_harness.rs @@ -725,7 +725,7 @@ async fn build_pipeline_inner( }; let bridge_path = shadow.bridge_socket().expect("bridge configured"); let bridge = Arc::new( - walshadow::bridge::connect_with_budget(bridge_path, Duration::from_secs(60)) + walshadow::bridge::connect_with_budget(bridge_path, 1, Duration::from_secs(60)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", bridge_path.display())), ); diff --git a/tests/oracle.rs b/tests/oracle.rs index e9fdb9ec..7c11cb0a 100644 --- a/tests/oracle.rs +++ b/tests/oracle.rs @@ -77,7 +77,7 @@ fn start_pg(tmp: &tempfile::TempDir, port: u16) -> Option { async fn oracle_on(sh: &Shadow) -> Oracle { let path = sh.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + let bridge = walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())); Oracle::new(Arc::new(bridge)) @@ -315,12 +315,14 @@ async fn worker_refuses_malformed_requests() { return; }; let socket = guard.sh.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(20)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(20)) .await .expect("bridge connect"); + // Past the bridge's frame prefix, as `encode_request` builds it let framed = |rows: u32, cols: u32, meta: &[u8], cells: &[u8]| -> Vec { - let mut v = rows.to_be_bytes().to_vec(); + let mut v = walshadow::bridge::request_frame(0); + v.extend_from_slice(&rows.to_be_bytes()); v.extend_from_slice(&cols.to_be_bytes()); v.extend_from_slice(meta); v.extend_from_slice(cells); @@ -363,7 +365,7 @@ async fn worker_refuses_malformed_requests() { ]; for (what, payload) in cases { let err = bridge - .encode_native(&payload) + .encode_native(payload) .await .err() .unwrap_or_else(|| panic!("{what}: worker accepted a malformed request")); @@ -376,7 +378,7 @@ async fn worker_refuses_malformed_requests() { // Reject bytes beyond declared cells let mut trailing = framed(1, 1, &one, &[0x01, 0, 0, 0, 4, 42, 0, 0, 0]); trailing.push(0); - assert!(bridge.encode_native(&trailing).await.is_err()); + assert!(bridge.encode_native(trailing).await.is_err()); // Parser errors preserve connection let oracle = Oracle::new(Arc::new(bridge)); diff --git a/tests/oracle_types_e2e.rs b/tests/oracle_types_e2e.rs index bf5cde9e..11f10739 100644 --- a/tests/oracle_types_e2e.rs +++ b/tests/oracle_types_e2e.rs @@ -101,7 +101,7 @@ async fn run_oracle_stats( // Worker binds only once recovery reaches consistency, so budget the dial let socket = shadow.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(30)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) .await .expect("bridge connect"); assert!( diff --git a/tests/runtime_config_e2e.rs b/tests/runtime_config_e2e.rs index 4e534ae3..7bdb0a90 100644 --- a/tests/runtime_config_e2e.rs +++ b/tests/runtime_config_e2e.rs @@ -478,7 +478,7 @@ async fn opt_in_non_empty_backfills_pre_opt_in_rows() { // `meta jsonb` sits outside the local matrix, so the backfill tail needs // the same oracle the WAL path uses let socket = shadow.bridge_socket().expect("bridge configured"); - let bridge = walshadow::bridge::connect_with_budget(socket, Duration::from_secs(30)) + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) .await .expect("bridge connect"); let oracle = Arc::new(walshadow::oracle::Oracle::new(Arc::new(bridge))); diff --git a/tests/shadow_catalog.rs b/tests/shadow_catalog.rs index 37bade07..e1b4909f 100644 --- a/tests/shadow_catalog.rs +++ b/tests/shadow_catalog.rs @@ -81,7 +81,7 @@ async fn open_catalog(shadow: &Shadow, replay_timeout: Duration) -> ShadowCatalo async fn open_bridge(shadow: &Shadow) -> Arc { let path = shadow.bridge_socket().expect("bridge configured"); Arc::new( - walshadow::bridge::connect_with_budget(path, Duration::from_secs(20)) + walshadow::bridge::connect_with_budget(path, 1, Duration::from_secs(20)) .await .unwrap_or_else(|e| panic!("bridge connect on {}: {e}", path.display())), ) diff --git a/tests/xact_buffer.rs b/tests/xact_buffer.rs index 7a05f936..1518ca9d 100644 --- a/tests/xact_buffer.rs +++ b/tests/xact_buffer.rs @@ -95,6 +95,7 @@ async fn open_catalog(shadow: &Shadow) -> ShadowCatalog { let bridge = Arc::new( walshadow::bridge::connect_with_budget( shadow.bridge_socket().expect("bridge configured"), + 1, Duration::from_secs(20), ) .await