From 50a84df717adc0327f9c5fcb593f7ae3fd6e38be Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:56:27 +0000 Subject: [PATCH 1/2] Bootstrap consistency PG base backup isn't consistent until after wal replay, stream wal in parallel to initial load Share WalReplaySink across bootstrap and wal replay Gate tuple visibility of base backup, avoids deleted rows from before base backup appearing Preserve TOAST chunk mirrors and make chunk selection deterministic Resume from oldest crossing transaction when replication slot permits Add --bootstrap-max-rate-kib for throttling --- Cargo.lock | 8 +- Cargo.toml | 2 +- architecture/README.md | 5 +- architecture/timeline_bootstrap.dot | 7 +- architecture/timeline_bootstrap.svg | 571 ++++++++-------- docs/getting-started.md | 5 +- docs/limitations.md | 15 + plans/GLOSSARY.md | 35 +- plans/INDEX.md | 2 +- plans/TOAST.md | 14 +- plans/bootstrap.md | 176 ++++- plans/future/INDEX.md | 1 + plans/future/bootstrap_open_xact_carry.md | 58 ++ src/backfill/backfill_bootstrap.rs | 81 ++- src/backfill/backfill_staging.rs | 4 +- src/backfill/backup_backfill.rs | 757 ++++++---------------- src/backfill/backup_page_walk.rs | 87 ++- src/backfill/bootstrap_window.rs | 531 +++++++++++++++ src/backfill/copy_backfill.rs | 122 ++-- src/backfill/mod.rs | 4 + src/backfill/visibility_gate.rs | 653 +++++++++++++++++++ src/backfill/visibility_repair.rs | 341 ++++++++++ src/backfill/wal_replay.rs | 591 +++++++++++++++++ src/bin/stream.rs | 519 +++++++++++---- src/catalog/shadow.rs | 2 +- src/catalog/shadow_catalog.rs | 33 +- src/decode/codecs.rs | 4 +- src/decode/visibility.rs | 103 ++- src/emit/pipeline/bootstrap.rs | 37 +- src/emit/pipeline/decode.rs | 6 +- src/emit/pipeline/tail.rs | 64 ++ src/emit/route.rs | 20 + src/filter/dirty_tree.rs | 2 +- src/filter/engine.rs | 4 +- src/lib.rs | 3 +- src/ops/preflight.rs | 26 + src/record.rs | 24 + src/schema.rs | 38 ++ src/source/catalog_capture.rs | 4 +- src/toast/resolver.rs | 15 +- src/xact/xact_buffer.rs | 23 +- tests/add_column_default.rs | 11 +- tests/bootstrap_crossing_xact_ch.rs | 143 ++++ tests/bootstrap_direct_ch.rs | 155 +---- tests/bootstrap_gate_ch.rs | 157 +++++ tests/bootstrap_object_store_ch.rs | 166 +---- tests/bootstrap_pipeline_ch.rs | 2 +- tests/bootstrap_toast_gate_ch.rs | 217 +++++++ tests/bootstrap_types_e2e.rs | 18 +- tests/bootstrap_window_ch.rs | 187 ++++++ tests/bootstrap_window_leg_ch.rs | 287 ++++++++ tests/common/bootstrap_ch_fixture.rs | 243 ++++++- tests/common/inproc_harness.rs | 30 +- tests/common/ports.rs | 21 + tests/composite_pkey.rs | 3 +- tests/control_plane_e2e.rs | 14 +- tests/copy_into.rs | 11 +- tests/ddl_replicates.rs | 26 +- tests/desc_log_e2e.rs | 9 +- tests/desc_log_restart_e2e.rs | 6 +- tests/dirty_admission_e2e.rs | 3 +- tests/foreign_database_e2e.rs | 3 +- tests/init_e2e.rs | 17 +- tests/kill_restart.rs | 11 +- tests/oracle_types_e2e.rs | 3 +- tests/pending_capture_e2e.rs | 3 +- tests/pgbench_acceptance.rs | 26 +- tests/pipeline_parallel_ddl_e2e.rs | 6 +- tests/pipeline_parallel_e2e.rs | 22 +- tests/runtime_config_e2e.rs | 33 +- tests/schema_evolution_cdc.rs | 3 +- tests/schema_evolution_pinned.rs | 3 +- tests/soft_delete_cdc.rs | 3 +- tests/source_reconnect.rs | 20 +- tests/subxact.rs | 14 +- tests/system_columns_cdc.rs | 3 +- tests/toast_e2e.rs | 3 +- tests/toast_rewrite_e2e.rs | 6 +- tests/toast_tombstone_e2e.rs | 3 +- tests/toast_truncate_drop_e2e.rs | 9 +- tests/truncate.rs | 11 +- tests/types_sweep.rs | 3 +- tests/visibility_repair_pg.rs | 301 +++++++++ tests/weird_identifiers.rs | 3 +- 84 files changed, 5474 insertions(+), 1741 deletions(-) create mode 100644 plans/future/bootstrap_open_xact_carry.md create mode 100644 src/backfill/bootstrap_window.rs create mode 100644 src/backfill/visibility_gate.rs create mode 100644 src/backfill/visibility_repair.rs create mode 100644 src/backfill/wal_replay.rs create mode 100644 tests/bootstrap_crossing_xact_ch.rs create mode 100644 tests/bootstrap_gate_ch.rs create mode 100644 tests/bootstrap_toast_gate_ch.rs create mode 100644 tests/bootstrap_window_ch.rs create mode 100644 tests/bootstrap_window_leg_ch.rs create mode 100644 tests/visibility_repair_pg.rs diff --git a/Cargo.lock b/Cargo.lock index 6e92a83f..767e5ed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2474,9 +2474,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -2874,9 +2874,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wal-rus" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b871b7189788c5b16df5357826d9f8bc83ad38114da3045124ebf445bdb783e" +checksum = "36245604722c122fed42a495fb0d0df572e559cd4b413dded7433fceeb744f2e" dependencies = [ "anyhow", "astral-tokio-tar", diff --git a/Cargo.toml b/Cargo.toml index 85eece44..0895b1bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ lz4 = ["clickhouse-c-rs/lz4"] zstd = ["clickhouse-c-rs/zstd"] [dependencies] -wal-rus = "0.3.1" +wal-rus = "0.3.2" clickhouse-c-rs = { version = "0.2", default-features = false, features = ["tokio", "tls"] } # hash keys are internal (relfilenodes, oids, xids, names), never network # input, so SipHash's HashDoS margin buys nothing at 2-4x per probe diff --git a/architecture/README.md b/architecture/README.md index d9dc5dca..70f584b4 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -34,8 +34,9 @@ derives off channel ① (cache miss → diff → `SchemaEvent` → ### 4. Bootstrap timeline — greenfield in five phases Catalog seed → BASE_BACKUP pump → drain to CH → shadow handoff → WAL -streaming. Bootstrap waits for CH writes, then uses backup end as new -restart point. First status update saves it in `manifest.toml`. +streaming. A concurrent WAL leg ships backup-window commits at their real +`_lsn`, above walked rows. Bootstrap waits for CH writes, then uses backup end +as restart point. First status update saves it in `manifest.toml`. ![bootstrap timeline](timeline_bootstrap.svg) diff --git a/architecture/timeline_bootstrap.dot b/architecture/timeline_bootstrap.dot index eb75e28a..7e6a482a 100644 --- a/architecture/timeline_bootstrap.dot +++ b/architecture/timeline_bootstrap.dot @@ -30,11 +30,13 @@ digraph timeline_bootstrap { lan2 [label="DiskLanderSink\ncatalog → shadow data dir\nuser-heap filenode Skip", fillcolor="#4D3A28"]; walk2 [label="PageWalkSink\ndecode 8 KiB heap pages\nmain + pg_toast tuples with TID\n→ BackfillTuple (mpsc)", fillcolor="#4D3A28"]; dsk2 [label="shadow data dir\ncatalog files landed", fillcolor="#4D3850", shape=note]; + leg2 [label="window WAL leg\ndaemon feed, sampled pre-backup\ndecode → insert tail", fillcolor="#4D3A28"]; bs2 -> src2 [label="replication protocol", color="#A1A9CC", dir=both, arrowtail=open]; bs2 -> lan2 [label="MultiplexSink"]; bs2 -> walk2 [label="MultiplexSink"]; lan2 -> dsk2 [label="land file"]; + src2 -> leg2 [label="window WAL, live", color="#A1A9CC", dir=both, arrowtail=open]; } map1 -> bs2 [label="open BASE_BACKUP", lhead=cluster_p2, color="#A1A9CC"]; @@ -43,12 +45,15 @@ digraph timeline_bootstrap { label="③ drain → ClickHouse (concurrent with ②)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; + gate3 [label="visibility gate\nhint bits → inline\nunknown → spool\nresolve from pg_xact + patch", fillcolor="#4D3A28"]; drain3 [label="pipeline::bootstrap::drain\nToastRows → mirror batch\nExternalToast → deferred\nflush chunks, fetch + resolve\nmap main rows; one seq per rfn flip", fillcolor="#4D3A28"]; ch3 [label="ClickHouse\nTOAST mirrors: put before deferred fetch\nmain tables: shared insert tail\nbatcher + inserter ×N + ack collector\ntail.finish → all seqs durable", fillcolor="#4D4128"]; + gate3 -> drain3 [label="tuples that were live\nat backup time"]; drain3 -> ch3 [label="BatcherMsg::Row", color="#BF8C5F"]; } - walk2 -> drain3 [label="BackfillTuple\n(bounded mpsc 256,\nbackpressures pump)", lhead=cluster_p3]; + walk2 -> gate3 [label="BackfillTuple\n(bounded mpsc 256,\nbackpressures pump)", lhead=cluster_p3]; + leg2 -> ch3 [label="window commits at real _lsn\noutrank walked rows", color="#BF8C5F", lhead=cluster_p3]; // ═════ ④ shadow handoff ═════ subgraph cluster_p4 { diff --git a/architecture/timeline_bootstrap.svg b/architecture/timeline_bootstrap.svg index 51a615bb..d7b30ca2 100644 --- a/architecture/timeline_bootstrap.svg +++ b/architecture/timeline_bootstrap.svg @@ -4,437 +4,478 @@ - - + + timeline_bootstrap - -walshadow greenfield bootstrap — 5 phases + +walshadow greenfield bootstrap — 5 phases cluster_p1 - -① catalog seed + +① catalog seed cluster_p2 - -② BASE_BACKUP pump — MultiplexSink fan-out + +② BASE_BACKUP pump — MultiplexSink fan-out cluster_p3 - -③ drain → ClickHouse  (concurrent with ②) + +③ drain → ClickHouse  (concurrent with ②) cluster_p4 - -④ shadow handoff + +④ shadow handoff cluster_p5 - -⑤ WAL streaming start  (→ steady-state) + +⑤ WAL streaming start  (→ steady-state) op1 - -walshadow-stream ---bootstrap-mode=direct ---ch-config=… + +walshadow-stream +--bootstrap-mode=direct +--ch-config=… map1 - -CatalogTracker::seed_from_source -builds CatalogMap + +CatalogTracker::seed_from_source +builds CatalogMap op1->map1 - - -spawn + + +spawn sql1 - -sidecar SQL on source -SELECT oid, relfilenode -FROM pg_class WHERE oid≥16384 + +sidecar SQL on source +SELECT oid, relfilenode +FROM pg_class WHERE oid≥16384 map1->sql1 - - - -libpq + + + +libpq bs2 - -BackupSource -(Direct | ObjectStore) -begin pump + +BackupSource +(Direct | ObjectStore) +begin pump - + map1->bs2 - - -open BASE_BACKUP + + +open BASE_BACKUP src2 - -BASE_BACKUP open -pg_export_snapshot() -start_lsn = X + +BASE_BACKUP open +pg_export_snapshot() +start_lsn = X bs2->src2 - - - -replication protocol + + + +replication protocol lan2 - -DiskLanderSink -catalog → shadow data dir -user-heap filenode Skip + +DiskLanderSink +catalog → shadow data dir +user-heap filenode Skip bs2->lan2 - - -MultiplexSink + + +MultiplexSink walk2 - -PageWalkSink -decode 8 KiB heap pages -main + pg_toast tuples with TID -→ BackfillTuple (mpsc) + +PageWalkSink +decode 8 KiB heap pages +main + pg_toast tuples with TID +→ BackfillTuple (mpsc) bs2->walk2 - - -MultiplexSink + + +MultiplexSink - + end4 - -BASE_BACKUP end -end_lsn = Y + +BASE_BACKUP end +end_lsn = Y - + bs2->end4 - - -finish + + +finish + + + +leg2 + +window WAL leg +daemon feed, sampled pre-backup +decode → insert tail + + + +src2->leg2 + + + +window WAL, live dsk2 - - - -shadow data dir -catalog files landed + + + +shadow data dir +catalog files landed lan2->dsk2 - - -land file + + +land file - - -drain3 - -pipeline::bootstrap::drain -ToastRows → mirror batch -ExternalToast → deferred -flush chunks, fetch + resolve -map main rows; one seq per rfn flip - - - -walk2->drain3 - - -BackfillTuple -(bounded mpsc 256, -backpressures pump) + + +gate3 + +visibility gate +hint bits → inline +unknown → spool +resolve from pg_xact + patch + + + +walk2->gate3 + + +BackfillTuple +(bounded mpsc 256, +backpressures pump) - + ch3 - -ClickHouse -TOAST mirrors: put before deferred fetch -main tables: shared insert tail -batcher + inserter ×N + ack collector -tail.finish → all seqs durable + +ClickHouse +TOAST mirrors: put before deferred fetch +main tables: shared insert tail +batcher + inserter ×N + ack collector +tail.finish → all seqs durable + + + +leg2->ch3 + + +window commits at real _lsn +outrank walked rows + + + +drain3 + +pipeline::bootstrap::drain +ToastRows → mirror batch +ExternalToast → deferred +flush chunks, fetch + resolve +map main rows; one seq per rfn flip + + + +gate3->drain3 + + +tuples that were live +at backup time - + drain3->ch3 - - -BatcherMsg::Row + + +BatcherMsg::Row - + out4 - -BootstrapOutcome -{start_lsn, end_lsn} + +BootstrapOutcome +{start_lsn, end_lsn} - + end4->out4 - - -end_lsn + + +end_lsn - + ctrl4 - - - -pg_control landed last -(barrier) + + + +pg_control landed last +(barrier) - + conf4 - -Shadow::enable_standby_recovery -append standby.signal + -restore_command + -primary_conninfo + +Shadow::enable_standby_recovery +append standby.signal + +restore_command + +primary_conninfo - + out4->conf4 - - + + - + seed5 - -use backup end as -new restart point + +use backup end as +new restart point - + out4->seed5 - - -next phase + + +next phase - + files4 - - - -postgresql.conf + -standby.signal + + + +postgresql.conf + +standby.signal - + conf4->files4 - - -write + + +write - + listen4 - -walsender listener up -(barrier before shadow start) + +walsender listener up +(barrier before shadow start) - + conf4->listen4 - - + + - + start4 - -Shadow::start (recovery mode) -block_in_place pg_ctl + +Shadow::start (recovery mode) +block_in_place pg_ctl - + listen4->start4 - - -barrier + + +barrier - + shd4 - -postmaster + walreceiver -begin replay + +postmaster + walreceiver +begin replay - + start4->shd4 - - -pg_ctl start + + +pg_ctl start - + apply5 - -pg_last_wal_replay_lsn ≥ end_lsn -ready for relation_at gate + +pg_last_wal_replay_lsn ≥ end_lsn +ready for relation_at gate - + shd4->apply5 - - -replay catches up + + +replay catches up - + feed5 - -SourceFeed open -START_REPLICATION -PHYSICAL <end_lsn> + +SourceFeed open +START_REPLICATION +PHYSICAL <end_lsn> - + seed5->feed5 - - + + - + manifest5 - - - -manifest.toml -saved restart state + + + +manifest.toml +saved restart state - + feed5->manifest5 - - -first status update + + +first status update - + repl5 - -streaming protocol -opens at end_lsn + +streaming protocol +opens at end_lsn - + feed5->repl5 - - - -START_REPLICATION + + + +START_REPLICATION - + hot5 - -hot streaming -(pipeline: reorder → decode ×M -→ same tail + DdlApplicator) -→ timeline_streaming + +hot streaming +(pipeline: reorder → decode ×M +→ same tail + DdlApplicator) +→ timeline_streaming - + feed5->hot5 - - + + - + legend - - - -node fill — actor - - - -operator / CLI - - - -source Postgres - - - -walshadow-stream - - - -on-disk artifact - - - -shadow Postgres - - - -ClickHouse - - -edge colour - -━━ - -physical replication protocol - -━━ - -libpq catalog query - -━━ - -ClickHouse Native blocks - -┄┄ - -shadow replay progress + + + +node fill — actor + + + +operator / CLI + + + +source Postgres + + + +walshadow-stream + + + +on-disk artifact + + + +shadow Postgres + + + +ClickHouse + + +edge colour + +━━ + +physical replication protocol + +━━ + +libpq catalog query + +━━ + +ClickHouse Native blocks + +┄┄ + +shadow replay progress diff --git a/docs/getting-started.md b/docs/getting-started.md index e85ca347..d816bedc 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -13,8 +13,9 @@ Prepare: - PostgreSQL account with `REPLICATION` and read access to selected tables - Matching PostgreSQL major for walshadow image -Source must use `wal_level = logical` and at least one WAL sender. Each selected -table needs a usable row key +Source must use `wal_level = logical` and at least two WAL senders for +concurrent streaming and base backup. Each selected table needs a usable row +key ```sql ALTER SYSTEM SET wal_level = logical; diff --git a/docs/limitations.md b/docs/limitations.md index e69547e2..d073d22f 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -47,6 +47,17 @@ request per insert batch ## Initial loads +- transactions open past greenfield handoff resume from their first buffered + record, using source or archived WAL; missing history stops replication + instead of skipping it. Earlier records can still leave missing inserts or + stale deletes +- mapped external TOAST values or unresolved multixacts trigger whole-relation + `COPY` repair; inline values and unmapped external columns stay on page walk +- repair reads each physical relation with `ONLY` and rejects row-security + filtering; source account must be able to read every row of repaired relations +- dropping or rewriting a relation before repair fails bootstrap +- DDL during greenfield bootstrap is unsupported; affected relations may be + skipped or fail repair - `copy` scans selected table through PostgreSQL SQL path - `base_backup` transfers cluster-sized backup even for one table - `object_store` requires full wal-g backup and continuous archived WAL to selection point @@ -59,6 +70,10 @@ Default `[toast] mode = "disabled"` cannot always reconstruct values stored externally before replication window. Enable ClickHouse TOAST storage before initial load when complete large-value history matters +Reused TOAST value IDs can leave ambiguous generations in backup chunk mirrors +when hint bits do not prove older chunks dead. COPY repair fixes baseline rows, +but later unchanged-pointer updates still depend on those mirrors + ## Not an HA system walshadow consumes PostgreSQL failover decisions, it does not make them. It diff --git a/plans/GLOSSARY.md b/plans/GLOSSARY.md index 478b4cb2..7df31bf4 100644 --- a/plans/GLOSSARY.md +++ b/plans/GLOSSARY.md @@ -56,9 +56,12 @@ once through `MultiplexSink`, catalog files land on shadow data dir while user-heap pages Tap through page walk into shared insert tail; handoff to WAL pump at `end_lsn` ([bootstrap.md](bootstrap.md)) +**backup window** — `[start_lsn, end_lsn]`, span covered by base-backup WAL +replay ([bootstrap.md](bootstrap.md)) + **BufferingDecoderSink** — decoder-side record sink: gates on catalog replay, decodes heap records into XactBuffer, intercepts TRUNCATE and -config writes. Shared by hot path and gap replay +config writes. Shared by hot path and WAL replay legs ([source.md](source.md), [decoder.md](decoder.md)) **catalog gate** — see wait_for_replay @@ -240,7 +243,7 @@ page walk and TOAST re-read, never the tuple-bytes path **gap replay** — `object_store` step fetching archive WAL `B_redo → S` into scratch, replaying committed rows at real commit LSNs -through shared decode path ([add_table.md](add_table.md)) +through `WalReplaySink` ([add_table.md](add_table.md)) **generation counter** — ShadowCatalog cache invalidation: single u64 bumped on any pg_class write. Coarse-fires by design, over-invalidates @@ -395,9 +398,9 @@ oracle as cells, neither is ever rendered locally reads and Native conversion over a unix socket ([oracle.md](oracle.md), [`pgext/walshadow.h`](../pgext/walshadow.h)) -**PgXactAccum / PgXactPatch** — backup-era `pg_xact` accumulated from -backup files, patched with commit/abort records harvested from gap-WAL -pre-scan; backs the visibility gate ([add_table.md](add_table.md)) +**PgXactAccum / PgXactPatch** — backup `pg_xact` state plus commit/abort +records harvested from gap or window WAL; backs visibility gate +([add_table.md](add_table.md), [bootstrap.md](bootstrap.md)) **pipeline** — parallel decode+insert tail: reorder → decode ×M → batcher → inserter ×N → ack watermark, in `src/pipeline/`; stands up @@ -619,17 +622,26 @@ buffer state is process-local and spill clears on boot `numeric(p,s)` maps `Decimal(p,s)` else String ([emitter.md](emitter.md)) -**visibility gate** — backup-page tuple filter (`src/visibility.rs`): -emit only when backup-era pg_xact says xmin committed and xmax -absent/aborted, infomask hint bits short-circuit; what makes backup -modes higher-fidelity than greenfield's raw walk -([add_table.md](add_table.md)) +**visibility gate** — backup-page tuple filter: emit when `xmin` committed and +`xmax` is absent or aborted; use hint bits before transaction logs. Greenfield +resolves deferred tuples after window replay. Drop TOAST chunks only when +proven dead +([add_table.md](add_table.md), [bootstrap.md](bootstrap.md)) + +**visibility repair** — read a relation through PostgreSQL `COPY` when backup +state cannot prove tuple visibility or a mapped value uses external TOAST. +Inline values stay on page walk; repair tags rows at `S` +([bootstrap.md](bootstrap.md)) **wait_for_replay / catalog gate** — ShadowCatalog gate blocking until `pg_last_wal_replay_lsn() >= commit_lsn`; enforces ordering invariant shadow replay LSN ≥ decoder read LSN so decoder reads post-DDL catalog ([shadow.md](shadow.md), [overview.md](overview.md)) +**WalReplaySink** — bounded WAL-range decoder used by greenfield window replay +and `object_store` gap replay ([bootstrap.md](bootstrap.md), +[add_table.md](add_table.md)) + **wal-rus** — WAL parsing / replication crate walshadow builds on: replication client and server halves, record parser, BASE_BACKUP and object-store fetch primitives @@ -645,6 +657,9 @@ threshold, shadow catches up via restore_command walker (cross-page stitching, 16 MiB segment buffer), fires per-record dispatch through sink fan-out ([source.md](source.md)) +**window leg** — greenfield WAL path covering backup window from live stream +or hydrated segments ([bootstrap.md](bootstrap.md)) + **XactBuffer** — per-xid hold-and-flush of decoded heaps + TOAST chunks until commit/abort, spilling past budget; PG `ReorderBuffer` analogue minus snapshot building (catalog state lives in shadow) diff --git a/plans/INDEX.md b/plans/INDEX.md index f67b866f..e297e512 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -36,7 +36,7 @@ components. User workflows and supported behavior live under `type_bridge`, synthetic columns, `DdlApplicator`, barrier fence - [bootstrap.md](bootstrap.md) — greenfield BASE_BACKUP, `BackupSource` / `BackupSink` traits, `MultiplexSink`, `PageWalkSink` 2A decoder, - shared insert tail, restart source fallback contract + shared insert tail, window WAL replay, restart fallback - [ops.md](ops.md) — retention, manifest floor, standby-status triple, resume invariants - [failover.md](failover.md) — source timeline crossing: frozen pause diff --git a/plans/TOAST.md b/plans/TOAST.md index fa33c1f3..7de5ecff 100644 --- a/plans/TOAST.md +++ b/plans/TOAST.md @@ -106,13 +106,13 @@ Reclamation is `ReplacingMergeTree` merge behavior, not walshadow logic. (`src/pipeline/bootstrap.rs`), unmapped rels drop at `lookup_mapping` (counted `unsupported_relations`), and with nothing mapped nothing defers, so the deferred hard-error path cannot fire. Bootstrap consults the static - `[table.*]` map only, never `[namespace.*]`. Filtering is at the drain, not - the walk: catalog seeding is unconditional, main heap pages still decode - and drop; only `pg_toast_*` decode is config-gated (`store_toast`). Seeds - every toast rel in the catalog — no per-table filter. A re-seed of a live - deployment runs as a one-off against a scratch - `--bootstrap-shadow-data-dir` and separate `--out-dir` so the live - manifest stays untouched. + `[table.*]` map only, never `[namespace.*]`. Catalog seeding is + unconditional and every toast rel seeds its mirror — no per-table filter; + only `pg_toast_*` decode is config-gated (`store_toast`). Main files + outside the mapping snapshot are declined at the walk, so a mirror-only + seed decodes chunk pages alone. A re-seed of a live deployment runs as a + one-off against a scratch `--bootstrap-shadow-data-dir` and separate + `--out-dir` so the live manifest stays untouched. - **Decode shape (R2).** Value reassembled before the main-table INSERT, stored inline `Bytea`/`Text`; `encode_value` (`src/ch_emitter.rs`) needs no toast-specific handling. Tier 3 detoast routing: `detoasted_value` runs diff --git a/plans/bootstrap.md b/plans/bootstrap.md index 24084864..dcc19718 100644 --- a/plans/bootstrap.md +++ b/plans/bootstrap.md @@ -4,7 +4,7 @@ Greenfield initial-attach. Streams source PG's `BASE_BACKUP` through one MultiplexSink fanning simultaneously onto shadow PG's data dir (catalog seed) and the shared pipeline insert tail (heap-data initial load — see [emitter.md](emitter.md)). Single pass over backup bytes, -no on-disk spool, no second BASE_BACKUP, no shadow-side user-heap +bounded tuple spools, no second BASE_BACKUP, no shadow-side user-heap landing ## Purpose @@ -37,16 +37,18 @@ for rendered diagram. Five clusters top→bottom: snapshot via `seed_in_snapshot` so concurrent DDL during seed read does not tear 2. **BASE_BACKUP pump** — `BackupSource` (Direct or ObjectStore) opens - backup; `MultiplexSink` dispatches each `FileMeta`: + backup; the backup-window WAL leg (below) starts alongside it; + `MultiplexSink` dispatches each `FileMeta`: - catalog filenodes & system files → `DiskLanderSink` (Keep) → written to shadow `data_dir` - - user heap → `PageWalkSink` (Tap) → decoded 8 KiB at a time + - user heap → `PageWalkSink` (Tap) → decoded 8 KiB at a time, less + the main files the caller declined (unmapped or initial-load opt-out) - denylist contents → Skip; denylist dir entries themselves → Keep as empty dirs 3. **Drain → CH** (concurrent with step 2) — `PageWalkSink` ships `BackfillTuple`s through bounded mpsc (`BOOTSTRAP_TUPLE_CHANNEL_CAP - = 256`, backpressures the tar pump) to - `pipeline::bootstrap::drain`, which synthesizes rows + = 256`, backpressures the tar pump) through the visibility gate + (below) to `pipeline::bootstrap::drain`, which synthesizes rows `{ op=Insert, commit_lsn=start_lsn }` against the snapshot `CatalogMap` and routes into the shared insert tail (batcher + inserter pool + ack collector, same unit as streaming — see @@ -66,7 +68,9 @@ for rendered diagram. Five clusters top→bottom: (see [shadow.md](shadow.md)). Daemon empties `postgresql.auto.conf`, starts shadow with `start_with_floor_retry`, waits for `end_lsn` with `wait_for_replay`, then supervises it (see [shadow.md](shadow.md)) -5. **Manifest + WAL pump start** — the ack atomic seeds at `end_lsn` +5. **Manifest + WAL pump start** — the window leg has sealed its tail by + here, so the range below `end_lsn` is durable on CH before the pump + takes over. The ack atomic seeds at `end_lsn` (first status tick persists it to `manifest.toml`), `SourceFeed` opens `START_REPLICATION PHYSICAL `, steady-state emitter (now backed by live `ShadowCatalog`) takes over. @@ -76,6 +80,101 @@ for rendered diagram. Five clusters top→bottom: Phases 1-3 run synchronously inside `run_bootstrap`; phases 4-5 hand off to daemon's main loop +## Backup-window WAL leg + +`BASE_BACKUP` produces a mix of page states across `[start_lsn, end_lsn]`. +Shadow recovery settles that mix with WAL. Window leg does same for +ClickHouse, emitting commits at real LSNs above walked rows tagged at +`start_lsn`. + +WAL comes from one of two places: + +- **Direct:** stream from source position sampled before `BASE_BACKUP`, using + daemon's replication connection without claiming its slot +- **Archive or object store:** replay WAL hydrated into shadow `pg_wal/` + +Direct mode falls back to landed WAL when live streaming fails. Bootstrap +fails if both paths fail. + +Read starts at segment boundary below sampled position. Commits through +sampled position are ignored because walked pages cover them. + +Leg reads through `end_lsn`, updates `PgXactPatch`, then waits within a bound +for transactions opened below handoff. Pump overlap is deduplicated by commit +LSN. + +If transactions remain open, `open_floor` reports their earliest buffered +record and pump resumes there. Initial manifest persists this lower position. +Source or archive must retain required WAL; a slot provides retention, but its +absence does not prove WAL unavailable. Missing history stops replication +through existing source/archive recovery path + +Window decoding uses pre-backup descriptors. Unknown filenodes increment +`unknown_rfns`; DDL during bootstrap remains unsupported. + +Live leg leaves its connection in `COPY`; replacement connection must match +sampled system ID and timeline. + +Direct mode needs two WAL senders for base backup and window leg. Metrics-only +mode skips leg. + +## Visibility gate + +Page walk sees dead, aborted, and in-flight versions. Visibility gate uses +hint bits and backup transaction logs to select live tuples. + +Gate sits between page walk and drain (`visibility_gate::stream_phase`, +shared with backup-sourced per-table loads). Emit when `xmin` committed and +`xmax` is absent or aborted. + +Unsettled tuples enter `DeferredSpool` and resolve after backup lands. Reading +partial `pg_xact` earlier could classify a committed deleter as in flight. + +Resolution runs after the backup-window leg, not at walk end +(`visibility_gate::resolve_greenfield`): + +- read landed `pg_xact` and `pg_multixact` +- overlay window commits from `PgXactPatch` +- ship survivors on a new tail at `_lsn = start_lsn` + +Window leg re-emits tuples committed inside window. Transactions open past +handoff remain limited, see V1 limits. + +TOAST chunks use a one-sided gate: drop only when hint bits prove them dead. +Chunk TID breaks same-LSN ties between reused value IDs. + +## Visibility repair + +When backup `pg_multixact` cannot decide visibility, gate marks whole relation +pending. `visibility_repair::repair` replaces walked tuples with visible, +detoasted rows from PostgreSQL `COPY`. + +An external TOAST pointer in a mapped column marks its relation pending during +page walk, unless hint bits already prove tuple dead. Owning TOAST storage alone +does not trigger repair: inline values and unmapped external columns use walked +tuples. A `(value_id, chunk_seq)` mirror cannot distinguish reused generations + +Pending covers the mapping snapshot only. The drain routes by mapping, so a +`COPY` of an unmapped relation ships rows it drops. + +Walk declines main files of unmapped and initial-load opt-out relations. +Once a relation becomes pending, gate discards subsequent main tuples. Earlier +walked rows and repaired rows share coverage LSN; covered WAL outranks both + +Chunk pages still seed mirror for later WAL rows carrying unchanged external +pointers. Missing mirror data remains fatal. + +Repair uses `FROM ONLY` to preserve physical relation ownership and +`row_security = off` to reject filtered snapshots. This setting detects policy +filtering, it does not grant permission to bypass policies + +Repaired rows use `start_lsn`, ensuring covered WAL outranks baseline rows. + +Dropped, rewritten, or reshaped relations fail repair under DDL-quiesce +contract. + +Per-table loads stop instead of repairing. Metrics-only runs skip gate. + ## Bootstrap oracle — tier-3 resolution The page-walk yields raw on-disk Datums, so oracle-routed types (jsonb, @@ -326,8 +425,9 @@ forks route identically matrix in diagram above. Lander never asks for `chunk()` (only Keeps or Skips). Tap sink can decline a user-heap entry by returning `Skip`, in which case body drops unread — `PageWalkSink::begin` does this for -`pg_control` etc that arrive at user-heap-looking paths or for files -whose path does not parse as `base//` +`pg_control` etc that arrive at user-heap-looking paths, for files +whose path does not parse as `base//`, and for main files +the caller declined up front (`with_skip_filenodes`) Stats recovery: orchestrator holds two `Arc` clones to same `Mutex>` — one typed for stats teardown & @@ -370,9 +470,15 @@ backfill row tags identically V1 limits: - **No FPI replay on backup pages.** Pages with `pd_lsn < start_lsn` - captured mid-write walk as they land in the backup. WAL in - `[start_lsn, end_lsn]` updating same tuples re-emits at higher `_lsn` & - `ReplacingMergeTree(_lsn)` collapses duplicate + walk as they land in the backup; the window leg re-emits every tuple WAL + touched in `[start_lsn, end_lsn]` at its real commit LSN, and + `ReplacingMergeTree(_lsn)` keeps that over the walked copy +- **Writers open past the handoff.** A transaction still open when the + window leg stops can leave missing inserts or stale deletes from records + before window. Pump rebuilds in-window records from `open_floor`, requiring + source or archive retention. + [future/bootstrap_open_xact_carry.md](future/bootstrap_open_xact_carry.md) + keeps those tuples until the transaction settles - **TOAST-spilled columns resolve when a chunk store is configured.** Inline varlena decodes through the heap decoder; external pointers surface as `ColumnValue::ExternalToast`. With `[toast] mode != disabled` @@ -389,8 +495,8 @@ V1 limits: default `mode = disabled` an unresolved value NULL/default-fills and is counted, not rejected. Full chunk-storage design in [TOAST.md](TOAST.md) -- **No 2C CH-side COPY load.** PageWalkSink (2A) is the sole - initial-load path; see [Why not 2C](#why-not-2c-ch-side-copy-load) below +- **No exported-snapshot COPY baseline.** Page walk supplies baseline, with + source COPY repair for unresolved physical visibility Rfn contiguity buys seq economy, not correctness: `PageWalkSink` emits all rows for one rfn contiguously before moving on, so @@ -472,23 +578,33 @@ task errors return through `drain_backfill` future. Both must be is emitter rejection — `bootstrap drain: emitter rejected tuple` wraps inner `DecoderSinkError` with context -## Why not 2C CH-side COPY load - -BASEBACKUP.md's Use Case 2C is parallel `COPY` from source PG to CH, -coordinated against `pg_export_snapshot()` so COPY snapshot & -BASE_BACKUP's start checkpoint align. Bootstrap does not use it; -PageWalkSink (2A) is the sole initial-load path - -Why: 2C's per-OID binary-COPY adapter list (`decode_numeric_pgcopy_binary` -& peers) is a separate codec walshadow would carry forever, growing as -type coverage expands. 2A's outstanding items (FPI replay, TOAST chunk -decode, on-disk page → tuple projection) are WAL-decoder work emitter -needs anyway. One decoder vs two — 2A wins on maintenance cost - -PageWalkSink walks pages from BASE_BACKUP tar bytes; does not issue -`COPY` against source PG. Source-side load during bootstrap is purely -BASE_BACKUP duration (when using DirectSource) or zero (when using -ObjectStoreSource + sidecar catalog-seed connection) +## Removing source SQL + +Page walk avoids SQL data scans for inline values, including inline text, and +for unmapped external columns. COPY remains a correctness fallback for mapped +external pointers and unresolved multixacts. Each repaired relation adds a full +source scan, including when backup payload comes from object storage + +Removing automatic COPY requires a physical visibility proof for every +retained tuple and chunk generation. Backup SLRUs plus commit overlay do not +identify every reused TOAST generation, and TID order does not establish age. +Recovery-backed SLRUs, transaction outcome tracking, and retained chunk +birth/death metadata must replace that missing information. WAL replay must +also wait for required chunk history instead of racing page walk + +Removing all source SQL needs additional changes: + +- derive initial descriptors from landed catalogs, buffering user tuples until + descriptors become available +- provision OID-exact type oracle from recovered catalog files instead of + source `pg_dump` +- replace SQL slot creation and validation with replication protocol commands + where supported, preserving restart-position checks +- replace source preflight and runtime-configuration SQL probes or make their + dependencies explicit + +Keep shadow catalog-sized. Landing user heaps to make shadow serve COPY would +replace source scans with a full physical replica ## Shadow-as-source rejected diff --git a/plans/future/INDEX.md b/plans/future/INDEX.md index 46797e30..50c0df3a 100644 --- a/plans/future/INDEX.md +++ b/plans/future/INDEX.md @@ -16,6 +16,7 @@ rationale under `plans/` only when code cannot express it * [failover.md](failover.md) — beyond the switchover crossing in [../failover.md](../failover.md): unplanned promotion (transaction-state fence, overwrite contrecord), slotless pause windows, timeline-aware archive and base-backup replay * [sync_commit_witness.md](sync_commit_witness.md) — walshadow as RPO=0 durability standby * [two_phase_commit.md](two_phase_commit.md) — `XLOG_XACT_PREPARE` handling and gxid-keyed buffer +* [bootstrap_open_xact_carry.md](bootstrap_open_xact_carry.md) — persist and settle tuples blocked by transactions open across greenfield handoff * [ch_bounce_recovery.md](ch_bounce_recovery.md) — deeper re-emit-from-spill on retry-budget exhaustion * [pinned_ddl_baseline.md](pinned_ddl_baseline.md) — schema-event outcome must be a function of config + baseline, not cache warmth: CH-existence / persisted-baseline options for cross-restart consistency, drop detection across downtime, opt-in mapping vs republish * [coverage100.md](coverage100.md) — drive `cargo llvm-cov` line coverage toward 100%: tiered work list (pure units → fixtures → live e2e → hard tail) diff --git a/plans/future/bootstrap_open_xact_carry.md b/plans/future/bootstrap_open_xact_carry.md new file mode 100644 index 00000000..169b3133 --- /dev/null +++ b/plans/future/bootstrap_open_xact_carry.md @@ -0,0 +1,58 @@ +# bootstrap: carry tuples of open transactions + +Greenfield gate drops tuples whose deciding transaction remains open at +handoff. Inserts can remain absent and deletes can remain live. Slot replay +only repairs records retained inside backup window +([bootstrap.md](../bootstrap.md)). + +## Invariant that makes it fixable + +Backup checkpoint flushes changes below redo before file copy. Crossing +transactions therefore have: + +- records at or above redo, replayable from `open_floor` with retained WAL +- records below redo, represented by deferred walked tuples + +Persist deferred tuples until shadow observes transaction outcome. + +## Gate changes + +- Return `Defer` for in-progress `xmin`, `xmax`, or multixact updater +- Move deferred tuples into persistent carry spool and record xids +- Record undecided `(rfn, xid)` pairs for pending relations. Delay relation + repair until all recorded xids settle +- Keep per-table load behavior unchanged: `Undecidable::Abort` gates deferrals + +## Persistence + +Store beside bootstrap marker in shadow data directory: + +- `DeferredSpool` carry file +- manifest containing `start_lsn`, xids, and pending relations + +Write before removing marker; remove once carry and pending sets empty. Do not +use startup-cleared spill directory. + +## Settle task + +Run after handoff and on startup when carry state exists: + +1. Build `PgXactView` from shadow `pg_xact` and `pg_multixact` +2. Wait when no carried xid changed +3. Resolve newly settled tuples and relations; rewrite remaining carry + +Keep `start_lsn` tag. On commit, emit inserted versions and withhold deleted +ones. On abort, emit deleted versions and drop inserts. + +## What stays + +Keep `open_floor` and `BootstrapHandoff::resume_lsn`. In-window deletes still +need retained WAL because only WAL carries them. + +## Tests + +Unit: carry in-progress `xmin` and `xmax`; filter pending xid hints; round-trip +manifest; shrink spool after settlement. + +E2E: cross bootstrap without `--slot` using INSERT, UPDATE, and DELETE, cover +COMMIT and ROLLBACK. Restart once between handoff and outcome. diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index d049ce21..2d2364f6 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -41,6 +41,7 @@ use crate::backfill::backup_sink::{ use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, StartInfo}; use crate::decode::decoder_sink::TupleObserver; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; +use ahash::HashSet; #[derive(Debug, Clone)] pub struct BootstrapConfig { @@ -51,6 +52,9 @@ pub struct BootstrapConfig { /// `rel_node < 16384` bootstrap rule misses. Empty in greenfield where /// seed runs after bootstrap pub catalog_filenodes: CatalogFilenodes, + /// Main filenodes whose pages nothing downstream keeps, declined + /// before decode. See [`PageWalkSink::with_skip_filenodes`] + pub skip_filenodes: HashSet<(Oid, Oid)>, } impl BootstrapConfig { @@ -58,6 +62,7 @@ impl BootstrapConfig { Self { shadow_data_dir, catalog_filenodes: CatalogFilenodes::new(), + skip_filenodes: HashSet::default(), } } @@ -65,6 +70,11 @@ impl BootstrapConfig { self.catalog_filenodes = c; self } + + pub fn with_skip_filenodes(mut self, skip: HashSet<(Oid, Oid)>) -> Self { + self.skip_filenodes = skip; + self + } } #[derive(Debug, Clone)] @@ -119,7 +129,8 @@ pub fn spawn_greenfield_bootstrap( })?; let lander = DiskLanderSink::new(cfg.catalog_filenodes); - let page_walk = PageWalkSink::new(catalog_map, tx, store_toast); + let page_walk = + PageWalkSink::new(catalog_map, tx, store_toast).with_skip_filenodes(cfg.skip_filenodes); let mux = MultiplexSink::new(lander, page_walk); // Keep typed Arc beside erased trait-object Arc to recover stats @@ -321,47 +332,33 @@ pub async fn seed_in_snapshot(client: &Client) -> Result { } async fn fetch_replident(client: &Client, c: char, rel_oid: Oid) -> Result { - match c { - 'd' => { - let row = client - .query_opt( - "SELECT indkey::int2[] FROM pg_index \ - WHERE indrelid = $1 AND indisprimary = true LIMIT 1", - &[&rel_oid], - ) - .await?; - let pk_attnums = row.map(|r| r.get::<_, Vec>(0)); - Ok(ReplIdent::Default { pk_attnums }) - } - 'n' => Ok(ReplIdent::Nothing), - 'f' => { - // Capture the PK even under FULL so the CH ORDER BY uses it, not `_lsn`. - let row = client - .query_opt( - "SELECT indkey::int2[] FROM pg_index \ - WHERE indrelid = $1 AND indisprimary = true LIMIT 1", - &[&rel_oid], - ) - .await?; - let pk_attnums = row.map(|r| r.get::<_, Vec>(0)); - Ok(ReplIdent::Full { pk_attnums }) - } - 'i' => { - let row = client - .query_one( - "SELECT indexrelid::oid, indkey::int2[] FROM pg_index \ - WHERE indrelid = $1 AND indisreplident = true LIMIT 1", - &[&rel_oid], - ) - .await - .context("bootstrap: replident='i' missing pg_index row")?; - Ok(ReplIdent::UsingIndex { - index_oid: row.get(0), - key_attnums: row.get(1), - }) - } - other => anyhow::bail!("bootstrap: unknown relreplident {other:?}"), - } + // Capture the PK even under FULL so the CH ORDER BY uses it, not `_lsn` + let pk_attnums = if c == 'd' || c == 'f' { + client + .query_opt( + "SELECT indkey::int2[] FROM pg_index \ + WHERE indrelid = $1 AND indisprimary = true LIMIT 1", + &[&rel_oid], + ) + .await? + .map(|r| r.get::<_, Vec>(0)) + } else { + None + }; + let using_index = if c == 'i' { + client + .query_opt( + "SELECT indexrelid::oid, indkey::int2[] FROM pg_index \ + WHERE indrelid = $1 AND indisreplident = true LIMIT 1", + &[&rel_oid], + ) + .await? + .map(|r| (r.get(0), r.get(1))) + } else { + None + }; + ReplIdent::from_parts(c, pk_attnums, using_index) + .map_err(|e| anyhow::anyhow!("bootstrap: {e} for relation {rel_oid}")) } async fn fetch_attributes(client: &Client, rel_oid: Oid) -> Result> { diff --git a/src/backfill/backfill_staging.rs b/src/backfill/backfill_staging.rs index 87b62c5c..c1ab8fbb 100644 --- a/src/backfill/backfill_staging.rs +++ b/src/backfill/backfill_staging.rs @@ -85,8 +85,8 @@ pub async fn prepare( let mut sess = StagingSession::connect(emitter).await?; // Freeze routing for entire staging plan let live_map = live.snapshot().await; - let mut staged: HashMap = HashMap::new(); - let mut rels = Vec::new(); + let mut staged: HashMap = HashMap::with_capacity(reqs.len()); + let mut rels = Vec::with_capacity(reqs.len()); for r in reqs { let name = &r.desc.rel_name; let Some(m) = live_map.get(name) else { diff --git a/src/backfill/backup_backfill.rs b/src/backfill/backup_backfill.rs index fec9f6af..5528026b 100644 --- a/src/backfill/backup_backfill.rs +++ b/src/backfill/backup_backfill.rs @@ -27,10 +27,8 @@ //! sentinel → fetch gap segments → records-only pre-scan (catalog-skew //! abort + [`PgXactPatch`] harvest) → filtered walk (gate resolves deferred //! tuples against backup pg_xact + patch at successful walk EOF) → gap replay -//! through the shared decode path ([`BufferingDecoderSink`] + -//! [`XactBuffer::drain_committed`]) with -//! rows emitted at real commit LSNs, commits past a rel's `S` dropped (the -//! live stream owns them; dedup absorbs overlap regardless). +//! through [`WalReplaySink`], shared with greenfield window replay. Rows use +//! commit LSNs and stop at each relation's coverage bound. //! //! The pre-scan aborts on gap writes that would invalidate the walk: a //! pg_class / pg_attribute new-tuple write whose row oid is (or cannot be @@ -59,34 +57,30 @@ use crate::backfill::backup_source::{BackupSink, BackupSource}; use crate::backfill::backup_source_direct::DirectSource; use crate::backfill::backup_source_object_store::ObjectStoreSource; use crate::backfill::spool::{DEFERRED_SPOOL_MEM_MAX, DeferredSpool}; -use crate::config::ResolvedConfig; -use crate::decode::heap_decoder::{CommittedTuple, XLOG_HEAP_OPMASK, XLOG_HEAP_TRUNCATE}; -use crate::decode::visibility::{ - PgMultiXactAccum, PgXactAccum, PgXactPatch, PgXactView, Visibility, tuple_visibility, +use crate::backfill::visibility_gate::{GateStats, Undecidable, resolve_phase, stream_phase}; +use crate::backfill::visibility_repair::PendingSet; +use crate::backfill::wal_replay::{ + ReplayStats, ReplayTargets, WalReplayInputs, WalReplaySink, pump_segments_through, }; +use crate::decode::heap_decoder::{XLOG_HEAP_OPMASK, XLOG_HEAP_TRUNCATE}; +use crate::decode::visibility::{PgMultiXactAccum, PgXactAccum, PgXactPatch, PgXactView}; use crate::decode::wal_xact::{ - XLOG_XACT_ABORT, XLOG_XACT_ABORT_PREPARED, XLOG_XACT_ASSIGNMENT, XLOG_XACT_COMMIT, - XLOG_XACT_COMMIT_PREPARED, XLOG_XACT_OPMASK, parse_xact_assignment, parse_xact_payload, + XLOG_XACT_ABORT, XLOG_XACT_ABORT_PREPARED, XLOG_XACT_COMMIT, XLOG_XACT_COMMIT_PREPARED, + XLOG_XACT_OPMASK, parse_xact_payload, }; -use crate::emit::ch_emitter::EmitterStats; -use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow}; -use crate::emit::pipeline::{Fatal, ack::AckHandle, bootstrap, tail}; +use crate::emit::pipeline::batcher::BatcherMsg; +use crate::emit::pipeline::tail::OwnedTail; +use crate::emit::pipeline::{Fatal, ack::AckHandle, bootstrap}; use crate::filter::main_data::{parse_xl_heap_truncate, parse_xl_relmap_update}; -use crate::filter::manifest::Manifest; use crate::filter::pg_class_decoder::{ DecodeOutcome, decode_pg_class_tuple, info_carries_new_tuple_heap, }; -use crate::mapping::MappingSnapshot; -use crate::record::{Record, RecordSink, SegmentSink, SinkError, WAL_SEG_SIZE}; +use crate::record::{Record, RecordSink, SinkError, segments_covering}; use crate::runtime_config::InitialLoadMode; use crate::schema::RelDescriptor; -use crate::source::wal_stream::WalStream; -use crate::toast::{ChunkRefMap, ToastResolver}; -use crate::xact::xact_buffer::{ - BufferingDecoderSink, DrainEntry, DrainedBatch, SubxactTracker, WalkStep, XactBuffer, - XactBufferConfig, detoast_heap, resolve_stash, -}; -use ahash::{HashMap, HashSet}; +use crate::toast::ToastResolver; +use crate::xact::xact_buffer::{XactBuffer, XactBufferConfig}; +use ahash::{HashMap, HashSet, HashSetExt}; /// Run one coalesced backup pass for `reqs` (all sharing `mode`). pub async fn run_pass( @@ -297,18 +291,17 @@ async fn walk_and_ship( // Dedicated tail: own CH connection, own seq space, own fatal — the // live pipeline never blocks on a backfill (Regime A) - let fatal = Fatal::new(); - let (msg_tx, ack, tail) = tail::spawn_with_config( + let tail = OwnedTail::spawn( &ctx.emitter, 1, ctx.stats.clone(), - Arc::new(crate::pos::Monotone::new(0)), - fatal.clone(), + Fatal::new(), ctx.config_rx.clone(), ctx.oracle.clone(), + "backup_backfill", ) .await - .map_err(|e| anyhow::anyhow!("backup_backfill: spawn insert tail: {e}"))?; + .map_err(anyhow::Error::msg)?; let pg_xact = Arc::new(std::sync::Mutex::new(PgXactAccum::new())); let pg_multixact = Arc::new(std::sync::Mutex::new(PgMultiXactAccum::new())); @@ -346,14 +339,14 @@ async fn walk_and_ship( gated_rx, filter, ctx.mapping.clone(), - msg_tx.clone(), - ack.clone(), + tail.msg_tx.clone(), + tail.ack.clone(), ctx.stats.clone(), resolver.clone(), DeferredSpool::new(toast_spool_path, DEFERRED_SPOOL_MEM_MAX), ctx.emitter.row_policy(), ctx.config_rx.as_ref().map(|rx| rx.borrow().clone()), - std::collections::HashSet::new(), + HashSet::new(), )); // Success signal before the joins: gate resolves deferred tuples only @@ -377,13 +370,14 @@ async fn walk_and_ship( let drain_join = drain.await.context("backup_backfill: drain join"); if let Err(e) = run_res { - quiesce_tail(msg_tx, ack, tail).await; + tail.quiesce().await; return Err(e); } - let gate_stats = match gate_join.and_then(|r| r.map_err(anyhow::Error::msg)) { + let (gate_stats, pg_xact_segments) = match gate_join.and_then(|r| r.map_err(anyhow::Error::msg)) + { Ok(s) => s, Err(e) => { - quiesce_tail(msg_tx, ack, tail).await; + tail.quiesce().await; return Err(e); } }; @@ -393,18 +387,18 @@ async fn walk_and_ship( outcome.rows_gated += gate_stats.gated; outcome.rows_deferred += gate_stats.deferred; outcome.multixact_emitted += gate_stats.multixact_emitted; - outcome.pg_xact_segments = gate_stats.pg_xact_segments; + outcome.pg_xact_segments = pg_xact_segments; let drain_outcome = match drain_join.and_then(|r| r.map_err(anyhow::Error::msg)) { Ok(o) => o, Err(e) => { - quiesce_tail(msg_tx, ack, tail).await; + tail.quiesce().await; return Err(e); } }; let mut next_seq = drain_outcome.next_seq; if let Some((segments, timeline, b_redo)) = replay { - let s_by_rfn: HashMap<(Oid, Oid), (Arc, u64)> = reqs + let s_by_rfn: ReplayTargets = reqs .iter() .map(|r| (rfn_key(&r.desc), (r.desc.clone(), r.s_lsn))) .collect(); @@ -415,8 +409,8 @@ async fn walk_and_ship( b_redo, s_by_rfn, resolver.clone(), - msg_tx.clone(), - ack.clone(), + tail.msg_tx.clone(), + tail.ack.clone(), next_seq, ) .await @@ -424,49 +418,20 @@ async fn walk_and_ship( let replay_stats = match replay_res { Ok(s) => s, Err(e) => { - quiesce_tail(msg_tx, ack, tail).await; + tail.quiesce().await; return Err(e); } }; next_seq = replay_stats.next_seq; outcome.rows_replayed = replay_stats.rows_replayed; - outcome.replay_commits_past_s = replay_stats.commits_past_s; + outcome.replay_commits_past_s = replay_stats.commits_past_through; } - tail.finish(msg_tx, ack, next_seq, &fatal) - .await - .map_err(anyhow::Error::msg)?; + tail.finish(next_seq).await.map_err(anyhow::Error::msg)?; Ok(()) } -/// Failed-pass teardown: with gate + drain already joined, dropping the -/// last producer handles closes the batcher, which final-flushes and -/// cascades the inserters + collector down. Bounded by the inserters' -/// retry policy (a CH outage trips their fatal, not a hang). -async fn quiesce_tail(msg_tx: mpsc::Sender, ack: AckHandle, tail: tail::TailParts) { - drop(msg_tx); - drop(ack); - tail.join().await; -} - -#[derive(Debug, Default)] -struct GateStats { - emitted: u64, - gated: u64, - deferred: u64, - multixact_emitted: u64, - pg_xact_segments: usize, -} - -/// Visibility gate between the page walk and the drain. Hint-decidable -/// tuples route immediately; undecidable ones (including every non-lock-only -/// multixact xmax) defer until walk EOF, when collected pg_xact + -/// pg_multixact (+ gap patch) are complete. Toast-chunk tuples bypass the -/// gate: the store is keyed and only referenced values get pulled. -/// Deferred resolution requires `walk_ok`: channel close alone also happens -/// when a failed source drops the sink mid-walk. An unresolvable multixact -/// errors the pass: emitting risks resurrecting a dead version whose delete -/// predates WAL coverage, skipping risks dropping a live row. +/// Run visibility gate and track `pg_xact` segments #[allow(clippy::too_many_arguments)] async fn gate_task( mut rx: mpsc::Receiver, @@ -477,85 +442,31 @@ async fn gate_task( patch: PgXactPatch, walk_ok: oneshot::Receiver<()>, mut deferred: DeferredSpool, -) -> Result { +) -> Result<(GateStats, usize), String> { let mut stats = GateStats::default(); - while let Some(t) = rx.recv().await { - if filter.is_toast(t.rfn.db_node, t.rfn.rel_node) { - if tx.send(t).await.is_err() { - return Ok(stats); - } - continue; - } - match tuple_visibility(t.xid, t.xmax, t.infomask, None) { - Visibility::Emit => { - if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { - stats.multixact_emitted += 1; - } - stats.emitted += 1; - if tx.send(t).await.is_err() { - return Ok(stats); - } - } - Visibility::Skip => stats.gated += 1, - Visibility::Defer => deferred - .push(t) - .await - .map_err(|e| format!("backup_backfill: deferred spool: {e}"))?, - Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), - } - } - // Walk EOF: sink dropped. Deferred tuples sit in the spool past its - // in-memory prefix, resident bytes bounded regardless of unhinted count. - stats.deferred = deferred.records(); - // pg_xact is complete only if the walk finished; a partial accum reads - // committed deleters as in-progress and recent aborts as ancient-committed, - // emitting dead tuples a rerun can't remove + // Per-table loads abort on unprovable tuples + let mut no_pending = PendingSet::empty(); + stream_phase( + &mut rx, + &tx, + &filter, + &mut no_pending, + &mut deferred, + &mut stats, + ) + .await?; if walk_ok.await.is_err() { stats.gated += stats.deferred; deferred.discard().await; - return Ok(stats); + return Ok((stats, 0)); } // Take the accums out so no std guard is held across the sends below let accum = std::mem::take(&mut *pg_xact.lock().expect("pg_xact accum lock")); let multi = std::mem::take(&mut *pg_multixact.lock().expect("pg_multixact accum lock")); - stats.pg_xact_segments = accum.segment_count(); + let segments = accum.segment_count(); let view = PgXactView::new(&accum, &patch).with_multixact(&multi); - let mut replay = deferred - .into_reader() - .await - .map_err(|e| format!("backup_backfill: deferred spool seal: {e}"))?; - while let Some(t) = replay - .next() - .await - .map_err(|e| format!("backup_backfill: deferred spool replay: {e}"))? - { - match tuple_visibility(t.xid, t.xmax, t.infomask, Some(&view)) { - Visibility::Emit => { - if t.infomask & crate::decode::visibility::HEAP_XMAX_IS_MULTI != 0 { - stats.multixact_emitted += 1; - } - stats.emitted += 1; - if tx.send(t).await.is_err() { - return Ok(stats); - } - } - Visibility::Skip | Visibility::Defer => stats.gated += 1, - Visibility::Unresolvable => return Err(unresolvable_multixact(&t)), - } - } - replay - .finish() - .await - .map_err(|e| format!("backup_backfill: deferred spool cleanup: {e}"))?; - Ok(stats) -} - -fn unresolvable_multixact(t: &BackfillTuple) -> String { - format!( - "backup_backfill: multixact xmax {} (rfn {}/{}) unresolvable from the backup's \ - pg_multixact snapshot; remedy: fresher backup, or initial_load='copy'", - t.xmax, t.rfn.db_node, t.rfn.rel_node - ) + resolve_phase(deferred, &view, &tx, Undecidable::Abort, &mut stats).await?; + Ok((stats, segments)) } // --------------------------------------------------------------------------- @@ -577,15 +488,10 @@ pub async fn fetch_gap_segments( tokio::fs::create_dir_all(seg_dir) .await .with_context(|| format!("create {}", seg_dir.display()))?; - let seg_size = WAL_SEG_SIZE; - let mut cur = SegmentName { - timeline, - log_id: (from >> 32) as u32, - seg_no: ((from & 0xFFFF_FFFF) / seg_size) as u32, - }; - let mut out = Vec::new(); - loop { - let name = cur.format(); + let segments = segments_covering(timeline, from..to.saturating_add(1)); + let mut out = Vec::with_capacity(segments.len()); + for seg in segments { + let name = seg.format(); let dst = seg_dir.join(&name); if !dst.exists() { walrus::pg::wal::fetch::handle( @@ -598,70 +504,75 @@ pub async fn fetch_gap_segments( .await .with_context(|| format!("fetch WAL {name}"))?; } - out.push((cur, dst)); - let seg_end = cur.start_lsn(seg_size).saturating_add(seg_size); - if to < seg_end { - break; - } - cur = cur.next(seg_size); + out.push((seg, dst)); } Ok(out) } -/// Segment output of the replay/pre-scan streams is discarded; only the -/// record dispatch matters. -struct DropSegments; - -impl SegmentSink for DropSegments { - fn on_segment<'a>( - &'a mut self, - _seg: SegmentName, - _bytes: &'a [u8], - _manifest: &'a Manifest, - ) -> Pin> + Send + 'a>> { - Box::pin(std::future::ready(Ok(()))) - } - - fn on_partial_segment<'a>( - &'a mut self, - _seg: SegmentName, - _bytes: &'a [u8], - _manifest: &'a Manifest, - ) -> Pin> + Send + 'a>> { - Box::pin(std::future::ready(Ok(()))) - } -} +// --------------------------------------------------------------------------- +// Gap replay +// --------------------------------------------------------------------------- -/// Drive fetched segments through a `RecordSink` in LSN order. Scoped to -/// the pass's database like the live stream, so replayed records defer -/// decode against the same catalog-dirty trees the hot path would build -async fn pump_segments_through( +/// Replay fetched gap segments between walk and live-stream coverage +#[allow(clippy::too_many_arguments)] +async fn replay_gap( + ctx: &PassContext, segments: &[(SegmentName, PathBuf)], timeline: u32, - target_db_oid: Oid, - sink: &mut (dyn RecordSink + Send), -) -> Result<()> { - let Some((first, _)) = segments.first() else { - return Ok(()); - }; - let mut stream = WalStream::new(timeline, WAL_SEG_SIZE, first.start_lsn(WAL_SEG_SIZE)) - .map_err(|e| anyhow::anyhow!("backup_backfill: WalStream: {e}"))?; - stream.filter_mut().set_target_db(target_db_oid); - let mut seg_sink = DropSegments; - for (seg, path) in segments { - let bytes = tokio::fs::read(path) + b_redo: u64, + targets: ReplayTargets, + resolver: ToastResolver, + msg_tx: mpsc::Sender, + ack: AckHandle, + next_seq: u64, +) -> Result { + let spill = ctx.scratch_dir.join("replay_spill"); + tokio::fs::create_dir_all(&spill).await.ok(); + let buffer = Arc::new(Mutex::new( + XactBuffer::new(XactBufferConfig::new(spill)) + .map_err(|e| anyhow::anyhow!("backup_backfill: replay xact buffer: {e}"))?, + )); + buffer.lock().await.clear_spill_dir().await.ok(); + + // Include opted-in main and TOAST filenodes + let mut filter_rfns: HashSet<(Oid, Oid)> = targets.keys().copied().collect(); + for (desc, _) in targets.values() { + if let Some(td) = ctx + .catalog + .lock() .await - .with_context(|| format!("read {}", path.display()))?; - stream - .push(seg.start_lsn(WAL_SEG_SIZE), &bytes, sink, &mut seg_sink) + .toast_descriptor_for(desc.oid) .await - .map_err(|e| anyhow::anyhow!("backup_backfill: replay {}: {e}", seg.format()))?; + .map_err(|e| anyhow::anyhow!("backup_backfill: toast descriptor: {e}"))? + { + filter_rfns.insert(rfn_key(&td)); + } } - stream - .close(None, sink) - .await - .map_err(|e| anyhow::anyhow!("backup_backfill: replay close: {e}"))?; - Ok(()) + + let mut sink = WalReplaySink::new(WalReplayInputs { + log: ctx.log.clone(), + buffer, + resolver, + filter_rfns, + targets, + from_lsn: b_redo, + // Filter is the opted-in set, so unfiltered rels are deliberate + whole_db_filter: false, + mapping: ctx.mapping.snapshot().await, + stats: ctx.stats.clone(), + budget: ctx.budget.clone(), + row_policy: ctx.emitter.row_policy(), + config: ctx.config_rx.as_ref().map(|rx| rx.borrow().clone()), + batch_rows: ctx.emitter.drain_batch_rows, + batch_bytes: ctx.emitter.drain_batch_bytes, + msg_tx, + ack, + next_seq, + // Pre-scan already harvested transaction patch + patch: None, + }); + pump_segments_through(segments, timeline, ctx.log.db_oid(), &mut sink).await?; + Ok(sink.stats()) } // --------------------------------------------------------------------------- @@ -722,6 +633,11 @@ struct PrescanSink { } impl PrescanSink { + /// Keep the first reason: later records observe a patch already known bad + fn fail_closed(&mut self, reason: String) { + self.skew.get_or_insert(reason); + } + fn observe(&mut self, record: &Record<'_>) { let rm = record.parsed.header.resource_manager_id; if rm == RmId::Xact as u8 { @@ -729,16 +645,20 @@ impl PrescanSink { let xid = record.parsed.header.xact_id; match info & XLOG_XACT_OPMASK { XLOG_XACT_COMMIT | XLOG_XACT_COMMIT_PREPARED => { - let payload = - parse_xact_payload(info, &record.parsed.main_data, record.page_magic) - .unwrap_or_default(); - self.patch.commit(xid, &payload.subxacts); + match parse_xact_payload(info, &record.parsed.main_data, record.page_magic) { + // COMMIT PREPARED: header xid is the finishing + // backend's, the verdict belongs to the prepared xid + Ok(p) => self + .patch + .commit(p.twophase_xid.unwrap_or(xid), &p.subxacts), + Err(e) => self.fail_closed(format!("malformed commit payload: {e}")), + } } XLOG_XACT_ABORT | XLOG_XACT_ABORT_PREPARED => { - let payload = - parse_xact_payload(info, &record.parsed.main_data, record.page_magic) - .unwrap_or_default(); - self.patch.abort(xid, &payload.subxacts); + match parse_xact_payload(info, &record.parsed.main_data, record.page_magic) { + Ok(p) => self.patch.abort(p.twophase_xid.unwrap_or(xid), &p.subxacts), + Err(e) => self.fail_closed(format!("malformed abort payload: {e}")), + } } _ => {} } @@ -838,361 +758,13 @@ impl RecordSink for PrescanSink { } } -// --------------------------------------------------------------------------- -// Gap replay -// --------------------------------------------------------------------------- - -struct ReplayStats { - next_seq: u64, - rows_replayed: u64, - commits_past_s: u64, -} - -/// Replay the gap through the shared decode path: heap records whose rfn is -/// in the filter set feed the same [`BufferingDecoderSink`] the hot path -/// uses (subxacts, TOAST reassembly, update/delete decode for free); commit -/// records drain through [`XactBuffer::drain_committed`] + -/// [`DrainedBatch::into_walk`] — the same apply plan the reorder barrier -/// runs — shipping rows at their real commit LSNs on the pass's tail. -#[allow(clippy::too_many_arguments)] -async fn replay_gap( - ctx: &PassContext, - segments: &[(SegmentName, PathBuf)], - timeline: u32, - b_redo: u64, - targets: HashMap<(Oid, Oid), (Arc, u64)>, - resolver: ToastResolver, - msg_tx: mpsc::Sender, - ack: AckHandle, - next_seq: u64, -) -> Result { - let spill = ctx.scratch_dir.join("replay_spill"); - tokio::fs::create_dir_all(&spill).await.ok(); - let buffer = Arc::new(Mutex::new( - XactBuffer::new(XactBufferConfig::new(spill)) - .map_err(|e| anyhow::anyhow!("backup_backfill: replay xact buffer: {e}"))?, - )); - buffer.lock().await.clear_spill_dir().await.ok(); - - // Filter rfns: opted-in mains + their toast rels (chunks reassemble - // inside the buffered xact) - let mut filter_rfns: HashSet<(Oid, Oid)> = targets.keys().copied().collect(); - for (desc, _) in targets.values() { - if let Some(td) = ctx - .catalog - .lock() - .await - .toast_descriptor_for(desc.oid) - .await - .map_err(|e| anyhow::anyhow!("backup_backfill: toast descriptor: {e}"))? - { - filter_rfns.insert(rfn_key(&td)); - } - } - - let mut sink = ReplaySink { - decoder: BufferingDecoderSink::new(ctx.log.clone(), buffer.clone()), - buffer, - log: ctx.log.clone(), - pending: Default::default(), - subxact_tracker: SubxactTracker::new(), - resolver, - filter_rfns, - targets, - b_redo, - mapping: ctx.mapping.snapshot().await, - stats: ctx.stats.clone(), - budget: ctx.budget.clone(), - row_policy: ctx.emitter.row_policy(), - config: ctx.config_rx.as_ref().map(|rx| rx.borrow().clone()), - batch_rows: ctx.emitter.drain_batch_rows, - batch_bytes: ctx.emitter.drain_batch_bytes, - msg_tx, - ack, - next_seq, - open: None, - rows_replayed: 0, - commits_past_s: 0, - }; - pump_segments_through(segments, timeline, ctx.log.db_oid(), &mut sink).await?; - - Ok(ReplayStats { - next_seq: sink.next_seq, - rows_replayed: sink.rows_replayed, - commits_past_s: sink.commits_past_s, - }) -} - -/// Serial gap drain over pre-filtered records; mirrors the daemon's -/// `DecoderXactPair` without the queueing worker. Rows gate per rel: one -/// seq per commit that routed at least one row, real `commit_lsn`, commits -/// at or under `b_redo` live in the walked backup pages, commits past the -/// rel's `S` belong to the live stream (dedup absorbs overlap regardless). -/// Catalog/config drain entries are ignored — the live stream owns DDL and -/// the prescan aborts on filtered-rel catalog skew below `S`. -struct ReplaySink { - decoder: BufferingDecoderSink, - buffer: Arc>, - log: Arc, - /// Always empty: gap replay reads committed history, where no - /// transaction is in flight to have speculative catalog state - pending: crate::catalog::pending::PendingCatalog, - subxact_tracker: SubxactTracker, - resolver: ToastResolver, - filter_rfns: HashSet<(Oid, Oid)>, - targets: HashMap<(Oid, Oid), (Arc, u64)>, - b_redo: u64, - /// Mapping version frozen at replay start: gap replay re-seeds route - /// state from current config, and no config event applies mid-replay - /// (catalog/config drain entries are ignored here), so one snapshot - /// covers the whole replay - mapping: MappingSnapshot, - stats: Arc, - budget: Option, - /// Boot-only delete-retention policy, frozen into route snapshots - row_policy: crate::emit::route::RowPolicy, - /// Config snapshot for route freezes: gap replay re-seeds from current - /// config, not history (route history has no WAL position) - config: Option>, - /// Drain-slice budget, same knobs as the pipeline reorder - batch_rows: usize, - batch_bytes: usize, - msg_tx: mpsc::Sender, - ack: AckHandle, - next_seq: u64, - /// Current commit's `(seq, rows routed)`; registered lazily on its - /// first routed row so row-less commits consume no seq - open: Option<(u64, u64)>, - rows_replayed: u64, - commits_past_s: u64, -} - -impl ReplaySink { - async fn on_commit( - &mut self, - xid: u32, - info: u8, - record: &Record<'_>, - ) -> std::result::Result<(), SinkError> { - let payload = parse_xact_payload(info, &record.parsed.main_data, record.page_magic) - .unwrap_or_default(); - // Deferred resolution for filenodes invisible at record time; - // installs decode verdicts + `O - B` barriers ahead of the drain - resolve_stash( - &self.buffer, - &self.log, - &self.pending, - xid, - &payload.subxacts, - record.next_lsn, - self.resolver.stats_handle(), - ) - .await - .map_err(SinkError::from)?; - let mut drain = self - .buffer - .lock() - .await - .drain_committed( - xid, - payload.xact_time, - record.source_lsn, - &payload.subxacts, - self.resolver.stores_chunks(), - ) - .await - .map_err(SinkError::from)?; - while let Some(batch) = drain - .next_batch(self.batch_rows, self.batch_bytes, self.budget.as_ref()) - .await - .map_err(SinkError::from)? - { - self.apply_batch(batch, drain.commit_ts, drain.commit_lsn) - .await?; - } - drain.finish().await.map_err(SinkError::from)?; - if let Some((seq, rows)) = self.open.take() { - self.ack.placed(seq, rows); - } - self.subxact_tracker.forget_tree(xid); - Ok(()) - } - - async fn apply_batch( - &mut self, - batch: DrainedBatch, - commit_ts: i64, - commit_lsn: u64, - ) -> std::result::Result<(), SinkError> { - let walk = batch.into_walk(); - let ref_maps: Vec<&ChunkRefMap> = walk.chunks.iter().map(|g| g.map()).collect(); - // One spool per xact; generations sealed before spooling carry None - let spool = walk.chunks.iter().find_map(|g| g.spool()); - let mut rows_cursor = 0usize; - for step in walk.steps { - match step { - WalkStep::Rows { upto } => { - if upto > rows_cursor { - self.resolver - .put_row_refs(walk.new_rows.spool(), &walk.new_rows[rows_cursor..upto]) - .await - .map_err(|e| SinkError::Other(format!("toast store put: {e}")))?; - rows_cursor = upto; - } - } - // Live stream owns DDL/config apply - WalkStep::Event(DrainEntry::Catalog(_)) - | WalkStep::Event(DrainEntry::Config(_)) => {} - WalkStep::Event(DrainEntry::ToastBarrier { - toast_relid, - marker_lsn, - }) => { - self.resolver - .rewrite_barrier(toast_relid, marker_lsn, commit_lsn) - .await - .map_err(|e| SinkError::Other(format!("toast rewrite barrier: {e}")))?; - } - WalkStep::Truncate(_) => { - // xl_heap_truncate carries no block ref, never passes the - // rfn filter - debug_assert!(false, "TRUNCATE heap in gap replay"); - } - WalkStep::Heap(mut heap) => { - let rfn = heap.decoded.rfn; - let Some((rel, s_cap)) = self.targets.get(&(rfn.db_node, rfn.rel_node)) else { - continue; - }; - if commit_lsn <= self.b_redo { - // Backup pages already reflect this commit; the walked - // copy (tagged min(B_redo, S)) carries it - continue; - } - if commit_lsn > *s_cap { - self.commits_past_s += 1; - continue; - } - let policy = self - .row_policy - .for_rel(self.config.as_deref(), &rel.rel_name); - // Append-only destination (no delete marker): see - // `decode_and_route` - if policy.system.is_deleted.is_none() - && matches!(heap.decoded.op, crate::decode::heap_decoder::HeapOp::Delete) - { - self.stats - .deletes_discarded - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - } - let rel = rel.clone(); - let value_permit = detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) - .await - .map_err(SinkError::from)?; - let Some(mapping) = self.mapping.get(&rel.rel_name) else { - self.stats - .unsupported_relations - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - }; - let mapping = Arc::new(mapping.clone()); - let rules = self - .config - .as_ref() - .map_or_else(Arc::default, |rc| rc.column_rules.clone()); - let route = crate::emit::route::RouteSnapshot::freeze(mapping, rules, policy); - let seq = if let Some((seq, rows)) = &mut self.open { - *rows += 1; - *seq - } else { - let seq = self.next_seq; - self.next_seq += 1; - self.ack.register(seq, commit_lsn); - self.open = Some((seq, 1)); - seq - }; - self.msg_tx - .send(BatcherMsg::Row(RoutedRow { - seq, - rel, - route, - committed: CommittedTuple { - decoded: heap.decoded, - commit_ts, - commit_lsn, - }, - value_permit: value_permit.map(Arc::new), - })) - .await - .map_err(|_| { - SinkError::Other("backup_backfill: replay tail closed".into()) - })?; - self.rows_replayed += 1; - } - } - } - Ok(()) - } -} - -impl RecordSink for ReplaySink { - fn on_record<'a>( - &'a mut self, - record: &'a Record<'a>, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - let rm = record.parsed.header.resource_manager_id; - if rm == RmId::Heap as u8 || rm == RmId::Heap2 as u8 { - let in_filter = record.parsed.blocks.first().is_some_and(|b| { - let rel = b.header.location.rel; - self.filter_rfns.contains(&(rel.db_node, rel.rel_node)) - }); - if in_filter { - self.decoder.on_record(record).await?; - } - } else if rm == RmId::Xact as u8 { - let info = record.parsed.header.info; - let xid = record.parsed.header.xact_id; - match info & XLOG_XACT_OPMASK { - XLOG_XACT_COMMIT | XLOG_XACT_COMMIT_PREPARED => { - self.on_commit(xid, info, record).await?; - } - XLOG_XACT_ABORT | XLOG_XACT_ABORT_PREPARED => { - let payload = - parse_xact_payload(info, &record.parsed.main_data, record.page_magic) - .unwrap_or_default(); - self.buffer - .lock() - .await - .abort(xid, record.source_lsn, &payload.subxacts) - .await - .map_err(SinkError::from)?; - self.subxact_tracker.forget_tree(xid); - } - XLOG_XACT_ASSIGNMENT => { - // Hint for eviction policy; correctness rides on the - // commit / abort record's authoritative subxact list - if let Some((xtop, subs)) = parse_xact_assignment(&record.parsed.main_data) - { - self.subxact_tracker.assign(xtop, &subs); - } - } - _ => { - // PREPARE / INVALIDATIONS unhandled; xact stays - // buffered until COMMIT_PREPARED - } - } - } - Ok(()) - }) - } -} - #[cfg(test)] mod tests { use super::*; use crate::decode::visibility::{ HEAP_XMAX_INVALID, HEAP_XMAX_IS_MULTI, HEAP_XMIN_COMMITTED, HEAP_XMIN_INVALID, }; + use crate::decode::wal_xact::{XACT_XINFO_HAS_TWOPHASE, XLOG_XACT_HAS_INFO}; use crate::record::Route; use walrus::pg::walparser::{ BlockLocation, RelFileNode, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, @@ -1329,6 +901,61 @@ mod tests { ); } + /// `xl_xact_twophase` payload: xact_time, xinfo, then the prepared xid + fn two_phase_main_data(prepared: u32) -> Vec { + let mut md: Vec = 7i64.to_le_bytes().to_vec(); + md.extend_from_slice(&XACT_XINFO_HAS_TWOPHASE.to_le_bytes()); + md.extend_from_slice(&prepared.to_le_bytes()); + md + } + + /// Patch prepared xid rather than finishing backend xid + #[test] + fn prescan_keys_two_phase_verdicts_to_the_prepared_xid() { + let mut s = prescan(16400, 16400); + s.observe(&record( + RmId::Xact, + XLOG_XACT_COMMIT_PREPARED | XLOG_XACT_HAS_INFO, + 777, + two_phase_main_data(800), + None, + )); + s.observe(&record( + RmId::Xact, + XLOG_XACT_ABORT_PREPARED | XLOG_XACT_HAS_INFO, + 778, + two_phase_main_data(801), + None, + )); + assert!(s.skew.is_none()); + let accum = PgXactAccum::new(); + let view = PgXactView::new(&accum, &s.patch); + assert_eq!( + view.xid_status(800), + crate::decode::visibility::XidStatus::Committed + ); + assert_eq!( + view.xid_status(801), + crate::decode::visibility::XidStatus::Aborted + ); + assert_eq!(s.patch.len(), 2, "finishing backend's xid stays unpatched"); + } + + /// Reject truncated subxact payload + #[test] + fn prescan_fails_closed_on_malformed_xact_payload() { + let mut s = prescan(16400, 16400); + s.observe(&record( + RmId::Xact, + XLOG_XACT_COMMIT | XLOG_XACT_HAS_INFO, + 700, + vec![0u8; 10], + None, + )); + assert!(s.skew.is_some(), "malformed commit payload fails closed"); + assert!(s.patch.is_empty()); + } + #[test] fn prescan_aborts_on_filtered_pg_class_rewrite() { let mut s = prescan(16400, 16400); @@ -1623,7 +1250,7 @@ mod tests { drop(walk_tx); walk_ok_tx.send(()).unwrap(); - let stats = gate.await.unwrap().unwrap(); + let (stats, _segments) = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { got.push(t.xid); @@ -1680,7 +1307,7 @@ mod tests { drop(walk_tx); drop(walk_ok_tx); - let stats = gate.await.unwrap().unwrap(); + let (stats, _segments) = gate.await.unwrap().unwrap(); let mut got = Vec::new(); while let Some(t) = gated_rx.recv().await { got.push(t.xid); @@ -1743,7 +1370,7 @@ mod tests { drop(walk_tx); walk_ok_tx.send(()).unwrap(); - let stats = gate.await.unwrap().unwrap(); + let (stats, _segments) = gate.await.unwrap().unwrap(); assert!(gated_rx.recv().await.is_none(), "dead tuple must not emit"); assert_eq!(stats.deferred, 1, "multixact defers to EOF"); assert_eq!(stats.gated, 1); diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index 8e697228..350a3894 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -28,7 +28,7 @@ use crate::decode::heap_decoder::{ ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, decode_block_data, }; use crate::schema::RelDescriptor; -use ahash::{HashMap, HashMapExt}; +use ahash::{HashMap, HashMapExt, HashSet}; /// Heap page size, PG compile-time, identical to wal-rus `BLOCK_SIZE` pub const PAGE_BYTES: usize = 8192; @@ -162,6 +162,9 @@ pub struct PageWalkStats { pub files_walked: u64, /// Filenode absent from catalog map, typically a race against the seed pub files_skipped_unknown_filenode: u64, + /// Main files the caller declined up front: relations whose rows the + /// drain or visibility repair would replace + pub files_skipped_by_caller: u64, pub toast_files_observed: u64, pub pages_walked: u64, pub slots_seen: u64, @@ -357,6 +360,9 @@ pub struct PageWalkSink { /// backfills tag each rel with its own boundary; greenfield leaves /// this empty). Keyed `(db_node, rel_node)`. lsn_overrides: HashMap<(Oid, Oid), u64>, + /// Main filenodes to decline at `begin`, keyed `(db_node, rel_node)`. + /// Their bodies drain unread instead of paying page decode + skip: HashSet<(Oid, Oid)>, pub stats: PageWalkStats, /// Bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]): `chunk` is async, so a /// full channel awaits in `ship_tuple`, parking the source body read @@ -411,6 +417,7 @@ impl PageWalkSink { catalog, source_lsn: 0, lsn_overrides: HashMap::new(), + skip: HashSet::default(), stats: PageWalkStats::default(), out_tx: Some(out_tx), captured: Vec::new(), @@ -445,6 +452,15 @@ impl PageWalkSink { self } + /// Decline listed main filenodes before decode. Caller policy: rows a + /// mapping snapshot never routes, or a relation visibility repair reads + /// whole. `pg_toast_*` filenodes are never declined, the mirror seeds + /// from every one of them + pub fn with_skip_filenodes(mut self, skip: HashSet<(Oid, Oid)>) -> Self { + self.skip = skip; + self + } + /// Test-mode: emitted tuples land in `captured` instead of the mpsc #[cfg(test)] pub fn new_capturing(catalog: CatalogMap) -> Self { @@ -452,6 +468,7 @@ impl PageWalkSink { catalog, source_lsn: 0, lsn_overrides: HashMap::new(), + skip: HashSet::default(), stats: PageWalkStats::default(), out_tx: None, captured: Vec::new(), @@ -594,6 +611,10 @@ impl BackupSink for PageWalkSink { self.stats.files_seen += 1; let desc = self.catalog.get(f.db, f.filenode); let is_toast = self.catalog.is_toast(f.db, f.filenode); + if !is_toast && self.skip.contains(&(f.db, f.filenode)) { + self.stats.files_skipped_by_caller += 1; + return Ok(FileAction::Skip); + } if is_toast { self.stats.toast_files_observed += 1; } else if desc.is_some() { @@ -709,6 +730,28 @@ pub(crate) fn make_rel() -> RelDescriptor { } } +/// [`make_rel`] retargeted onto another relation. Kind follows the +/// namespace, the way PostgreSQL names chunk relations +#[cfg(test)] +pub(crate) fn make_rel_named( + oid: Oid, + rel_node: Oid, + toast_oid: Oid, + name: crate::schema::RelName, +) -> std::sync::Arc { + let mut desc = make_rel(); + desc.rfn.rel_node = rel_node; + desc.oid = oid; + desc.toast_oid = toast_oid; + desc.kind = if &*name.namespace == PG_TOAST_NS { + 't' + } else { + 'r' + }; + desc.rel_name = name; + std::sync::Arc::new(desc) +} + /// Test fixture: synthesise an 8 KiB heap page with one int4 tuple in /// PG on-disk layout. Shared with `backfill_bootstrap`. #[cfg(test)] @@ -971,6 +1014,48 @@ mod tests { assert_eq!(sink.stats.files_seen, 0); } + /// Caller-declined main files drain unread: the drain drops unmapped + /// rows and repair replaces pending ones. Chunk files still walk, the + /// mirror seeds from every toast rel + #[tokio::test] + async fn pagewalk_sink_declines_caller_skipped_main_files() { + use crate::schema::RelName; + let mut catalog = CatalogMap::new(); + catalog.insert(Arc::new(make_rel())); + let mut toast = make_rel(); + toast.rfn.rel_node = 16401; + toast.oid = 16401; + toast.rel_name = RelName::new("pg_toast", "pg_toast_16400"); + catalog.insert(Arc::new(toast)); + let mut sink = PageWalkSink::new_capturing_with_toast(catalog) + .with_skip_filenodes([(5, 16400), (5, 16401)].into_iter().collect()); + sink.start(&StartInfo { + start_lsn: 0x1000, + timeline: 1, + tablespaces: Vec::new(), + }) + .await + .unwrap(); + for (i, (path, want)) in [ + ("base/5/16400", FileAction::Skip), + ("base/5/16401", FileAction::Tap), + ] + .iter() + .enumerate() + { + let meta = FileMeta { + path: PathBuf::from(path), + size: PAGE_BYTES as u64, + mode: 0o600, + kind: FileKind::File, + }; + assert_eq!(sink.begin(EntryId(i as u64), &meta).await.unwrap(), *want); + } + assert_eq!(sink.stats.files_skipped_by_caller, 1); + assert_eq!(sink.stats.files_walked, 0); + assert_eq!(sink.stats.toast_files_observed, 1); + } + #[tokio::test] async fn pagewalk_sink_skips_when_filenode_absent_from_catalog() { let sink = PageWalkSink::new_capturing(CatalogMap::new()); diff --git a/src/backfill/bootstrap_window.rs b/src/backfill/bootstrap_window.rs new file mode 100644 index 00000000..4fb6c1cb --- /dev/null +++ b/src/backfill/bootstrap_window.rs @@ -0,0 +1,531 @@ +//! Replay greenfield backup-window WAL into ClickHouse +//! +//! [`stream_window`] reads live WAL beside `BASE_BACKUP`; [`replay_segments`] +//! reads hydrated segments. Both emit through [`WalReplaySink`] at commit LSN +//! so replayed rows outrank page-walk rows tagged at backup start + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use tokio::sync::{Mutex, watch}; +use walrus::pg::backup::format_pg_lsn; +use walrus::pg::wal::segment::SegmentName; +use walrus::pg::walparser::Oid; + +use crate::backfill::backup_page_walk::CatalogMap; +use crate::backfill::wal_replay::{ + DropSegments, ReplayStats, ReplayTargets, WalReplayInputs, WalReplaySink, pump_segments_through, +}; +use crate::catalog::desc_log::{BatchRecord, DescLogIdentity, DescriptorLog, LogEntry, LogValue}; +use crate::config::ResolvedConfig; +use crate::decode::visibility::PgXactPatch; +use crate::emit::ch_emitter::{EmitterConfig, EmitterStats}; +use crate::emit::pipeline::Fatal; +use crate::emit::pipeline::tail::OwnedTail; +use crate::mapping::MappingHandle; +use crate::record::{WAL_SEG_SIZE, segments_covering}; +use crate::source::source_feed::{SourceEvent, SourceFeed, StandbyStatus}; +use crate::source::wal_stream::WalStream; +use crate::toast::ToastResolver; +use crate::xact::xact_buffer::{XactBuffer, XactBufferConfig}; +use ahash::HashSet; + +/// Wind-down poll while source sends no WAL +const STOP_POLL: Duration = Duration::from_millis(100); + +/// Maximum wait for transactions open below handoff +const WIND_DOWN_MAX: Duration = Duration::from_secs(5); + +/// Inputs shared with concurrent bootstrap drain +#[derive(Clone)] +pub struct WindowLegConfig { + pub emitter: EmitterConfig, + pub mapping: MappingHandle, + pub config: Arc, + /// Shared emitter counters + pub stats: Arc, + /// Shared TOAST resolver + pub resolver: ToastResolver, + /// Bootstrap conversion oracle + pub oracle: Option>, + /// Shared pipeline failure state + pub fatal: Fatal, + /// Transaction spill and descriptor-log root + pub scratch_dir: PathBuf, + /// Commit overlay for walked-tuple visibility + pub patch: Arc>, + pub catalog: CatalogMap, + pub pg_major: u32, + pub system_id: String, + pub timeline: u32, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct WindowLegStats { + pub replay: ReplayStats, + /// Walk coverage floor, read starts at previous segment boundary + pub from_lsn: u64, + /// One past the last byte the leg covered + pub through_lsn: u64, + /// Lowest first-record LSN among transactions open at seal + pub open_floor: Option, +} + +/// Stream window on caller's slotless replication connection +/// +/// `stop` publishes `end_lsn`; zero stops at current position +pub async fn stream_window( + cfg: WindowLegConfig, + feed: &mut SourceFeed, + from_lsn: u64, + stop: watch::Receiver>, +) -> Result { + let leg = Leg::open(&cfg, from_lsn) + .await + .context("bootstrap window leg: open")?; + run_live(leg, cfg.timeline, feed, from_lsn, stop).await +} + +/// Replay window WAL already on disk +pub async fn replay_segments( + cfg: WindowLegConfig, + segments: &[(SegmentName, PathBuf)], + from_lsn: u64, + end_lsn: u64, +) -> Result { + let mut leg = Leg::open(&cfg, from_lsn) + .await + .context("bootstrap window leg: open")?; + let db_oid = leg.db_oid; + let drive = pump_segments_through(segments, cfg.timeline, db_oid, &mut leg.sink).await; + if let Err(e) = drive { + leg.quiesce().await; + return Err(e.context("bootstrap window leg: replay segments")); + } + leg.close(from_lsn, end_lsn, end_lsn).await +} + +/// Enumerate complete segment range covering `[from_lsn, end_lsn)`. A boundary +/// `end_lsn` needs no further segment, and `pg_basebackup` ships none +/// (XLByteToPrevSeg) +pub async fn segments_in_dir( + dir: &Path, + timeline: u32, + from_lsn: u64, + end_lsn: u64, +) -> Result> { + let segments = segments_covering(timeline, read_start(from_lsn)..end_lsn); + let mut out = Vec::with_capacity(segments.len()); + for seg in segments { + let path = dir.join(seg.format()); + anyhow::ensure!( + tokio::fs::try_exists(&path).await.unwrap_or(false), + "bootstrap window leg: WAL segment {} missing from {}", + seg.format(), + dir.display(), + ); + out.push((seg, path)); + } + Ok(out) +} + +/// Align read start below commit floor +fn read_start(from_lsn: u64) -> u64 { + WalStream::align_down(from_lsn, WAL_SEG_SIZE) +} + +/// Every catalog filenode decodes, so TOAST records reach their parent rows; +/// only non-TOAST rels route. No ceiling: a leg owns every commit it decodes, +/// the pump resumes at seal +fn replay_scope(catalog: &CatalogMap) -> (HashSet<(Oid, Oid)>, ReplayTargets) { + let filter_rfns = catalog + .descriptors() + .map(|d| (d.rfn.db_node, d.rfn.rel_node)) + .collect(); + let targets = catalog + .descriptors() + .filter(|d| !catalog.is_toast(d.rfn.db_node, d.rfn.rel_node)) + .map(|d| ((d.rfn.db_node, d.rfn.rel_node), (d.clone(), u64::MAX))) + .collect(); + (filter_rfns, targets) +} + +/// Replay sink and owned insert tail +struct Leg { + sink: WalReplaySink, + tail: OwnedTail, + db_oid: Oid, +} + +impl Leg { + async fn open(cfg: &WindowLegConfig, from_lsn: u64) -> Result { + let db_oid = cfg + .catalog + .descriptors() + .map(|d| d.rfn.db_node) + .find(|db| *db != 0) + .context("bootstrap window leg: catalog map names no database")?; + let log = seed_scratch_log( + &cfg.scratch_dir.join("desc_log"), + DescLogIdentity { + pg_major: cfg.pg_major, + system_id: cfg.system_id.clone(), + timeline: cfg.timeline, + db_oid, + wal_seg_size: WAL_SEG_SIZE as u32, + }, + &cfg.catalog, + read_start(from_lsn), + ) + .await?; + + let spill = cfg.scratch_dir.join("xact_spill"); + tokio::fs::create_dir_all(&spill) + .await + .with_context(|| format!("create {}", spill.display()))?; + let buffer = Arc::new(Mutex::new( + XactBuffer::new(XactBufferConfig::new(spill)) + .map_err(|e| anyhow::anyhow!("bootstrap window leg: xact buffer: {e}"))?, + )); + buffer.lock().await.clear_spill_dir().await.ok(); + + let (filter_rfns, targets) = replay_scope(&cfg.catalog); + let tail = OwnedTail::spawn( + &cfg.emitter, + 1, + cfg.stats.clone(), + cfg.fatal.clone(), + None, + cfg.oracle.clone(), + "bootstrap window leg", + ) + .await + .map_err(anyhow::Error::msg)?; + + let sink = WalReplaySink::new(WalReplayInputs { + log, + buffer, + resolver: cfg.resolver.clone(), + filter_rfns, + targets, + from_lsn, + // Unknown user filenodes imply window DDL + whole_db_filter: true, + mapping: cfg.mapping.snapshot().await, + stats: cfg.stats.clone(), + budget: cfg.resolver.budget().cloned(), + row_policy: cfg.emitter.row_policy(), + config: Some(cfg.config.clone()), + batch_rows: cfg.emitter.drain_batch_rows, + batch_bytes: cfg.emitter.drain_batch_bytes, + msg_tx: tail.msg_tx.clone(), + ack: tail.ack.clone(), + next_seq: 0, + patch: Some(cfg.patch.clone()), + }); + Ok(Self { sink, tail, db_oid }) + } + + /// Flush and prove every sequence durable + /// + /// `handoff` bounds the open-xact report: the pump rebuilds anything + /// opened above it on its own + async fn close(self, from_lsn: u64, through_lsn: u64, handoff: u64) -> Result { + let replay = self.sink.stats(); + let open_floor = self + .sink + .xacts_opened_below(handoff) + .await + .into_iter() + .map(|(_, first_lsn)| first_lsn) + .min(); + drop(self.sink); + self.tail + .finish(replay.next_seq) + .await + .map_err(anyhow::Error::msg)?; + Ok(WindowLegStats { + replay, + from_lsn, + through_lsn, + open_floor, + }) + } + + async fn quiesce(self) { + drop(self.sink); + self.tail.quiesce().await; + } +} + +/// Seed window-wide descriptors from snapshot catalog +async fn seed_scratch_log( + dir: &Path, + identity: DescLogIdentity, + catalog: &CatalogMap, + read_start: u64, +) -> Result> { + tokio::fs::create_dir_all(dir) + .await + .with_context(|| format!("create {}", dir.display()))?; + for f in [ + crate::catalog::desc_log::CKPT_FILE, + crate::catalog::desc_log::TAIL_FILE, + ] { + let _ = tokio::fs::remove_file(dir.join(f)).await; + } + let log = DescriptorLog::open(dir, identity) + .await + .context("bootstrap window leg: open scratch descriptor log")?; + let entries = catalog + .descriptors() + .map(|d| { + Arc::new(LogEntry { + valid_from: read_start, + oid: d.oid, + rfn: d.rfn, + value: LogValue::Present(d.clone()), + }) + }) + .collect(); + log.seed( + BatchRecord { + captured_at: read_start, + commit_lsn: 0, + observations: Vec::new(), + ambiguities: Vec::new(), + entries, + }, + read_start, + ) + .await + .context("bootstrap window leg: seed scratch descriptor log")?; + Ok(Arc::new(log)) +} + +/// Consume live WAL through published `end_lsn` +async fn run_live( + mut leg: Leg, + timeline: u32, + feed: &mut SourceFeed, + from_lsn: u64, + mut stop: watch::Receiver>, +) -> Result { + let begin = read_start(from_lsn); + feed.start_physical_replication(None, begin, timeline) + .await + .context("bootstrap window leg: START_REPLICATION")?; + let mut stream = WalStream::new(timeline, WAL_SEG_SIZE, begin) + .map_err(|e| anyhow::anyhow!("bootstrap window leg: WalStream: {e}"))?; + stream.filter_mut().set_target_db(leg.db_oid); + let mut seg_sink = DropSegments; + let mut buf = Vec::new(); + + let mut wind_down: Option = None; + let res = loop { + // Read through end_lsn so final-segment commits reach PgXactPatch + let target = *stop.borrow_and_update(); + if let Some(target) = target.filter(|t| stream.next_lsn().get() >= *t) { + let spanning = leg.sink.xacts_opened_below(target).await; + if spanning.is_empty() { + break Ok(()); + } + let since = *wind_down.get_or_insert_with(std::time::Instant::now); + if since.elapsed() >= WIND_DOWN_MAX { + // Pump resumes from open_floor to rebuild these transactions + tracing::warn!( + target: "walshadow::bootstrap", + xids = ?spanning.iter().map(|(xid, _)| *xid).collect::>(), + target = %format_pg_lsn(target), + "backup-window leg stopped waiting on xacts open across the \ + handoff; the pump resumes below their first records instead", + ); + break Ok(()); + } + } + // Slotless status update only prevents sender timeout + let status = StandbyStatus::collapsed(stream.next_lsn().get()); + let event = tokio::select! { + biased; + // Closed watch means caller is unwinding + res = stop.changed() => match res { + Ok(()) => continue, + Err(_) => break Ok(()), + }, + // Poll only while waiting for open transactions + _ = tokio::time::sleep(STOP_POLL), if wind_down.is_some() => continue, + res = feed.next_event(status, &mut buf) => res, + }; + match event { + Ok(SourceEvent::Wal(chunk)) => { + let (lsn, data) = (chunk.start_lsn, chunk.data); + if let Err(e) = stream.push(lsn, data, &mut leg.sink, &mut seg_sink).await { + break Err(anyhow::anyhow!("bootstrap window leg: push: {e}")); + } + } + Ok(SourceEvent::TimelineEnd) => { + break Err(anyhow::anyhow!( + "bootstrap window leg: source ended timeline {timeline} inside the \ + backup window; the source was promoted mid-bootstrap" + )); + } + Ok(SourceEvent::Shutdown) => { + break Err(anyhow::anyhow!( + "bootstrap window leg: source closed the stream inside the backup window" + )); + } + Err(e) => break Err(e.context("bootstrap window leg: source read")), + } + }; + let through = stream.next_lsn().get(); + // Read runs past the handoff to land final-segment commits; report open + // xacts at the handoff the wind-down waited on + let handoff = stop.borrow().unwrap_or(through).min(through); + if let Err(e) = res { + leg.quiesce().await; + return Err(e); + } + leg.close(from_lsn, through, handoff).await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backfill::backup_page_walk::make_rel_named; + use crate::catalog::desc_log::LookupResult; + use crate::schema::{RelDescriptor, RelName}; + + /// `make_rel`'s database, the one `replay_scope` keys off + const DB: Oid = 5; + + fn desc(oid: Oid, rel_node: u32, namespace: &str, name: &str) -> Arc { + make_rel_named(oid, rel_node, 0, RelName::new(namespace, name)) + } + + fn ident() -> DescLogIdentity { + DescLogIdentity { + pg_major: 17, + system_id: "7300000000000000001".into(), + timeline: 1, + db_oid: DB, + wal_seg_size: WAL_SEG_SIZE as u32, + } + } + + /// Read starts at segment boundary below coverage floor + #[test] + fn read_start_aligns_below_the_floor() { + assert_eq!(read_start(WAL_SEG_SIZE), WAL_SEG_SIZE); + assert_eq!(read_start(WAL_SEG_SIZE + 1), WAL_SEG_SIZE); + assert_eq!(read_start(3 * WAL_SEG_SIZE - 1), 2 * WAL_SEG_SIZE); + } + + /// Snapshot descriptors cover full window + #[tokio::test] + async fn scratch_log_answers_window_lookups() { + let tmp = tempfile::tempdir().unwrap(); + let mut catalog = CatalogMap::new(); + let d = desc(16400, 16400, "public", "t"); + catalog.insert(d.clone()); + let from = 3 * WAL_SEG_SIZE + 4096; + let log = seed_scratch_log(tmp.path(), ident(), &catalog, read_start(from)) + .await + .unwrap(); + + assert_eq!(log.covered_through(), read_start(from)); + assert!(matches!( + log.descriptor_at(d.rfn, from), + LookupResult::Present(got) if got.rel_name == d.rel_name + )); + assert!( + matches!( + log.descriptor_at(d.rfn, read_start(from)), + LookupResult::Present(_) + ), + "records in the alignment prefix decode too; their commits drop on `from_lsn`", + ); + assert!(matches!( + log.descriptor_at(d.rfn, read_start(from) - 1), + LookupResult::NotCovered + )); + } + + /// TOAST records decode but do not route directly + #[test] + fn toast_rels_filter_but_do_not_target() { + let mut catalog = CatalogMap::new(); + catalog.insert(desc(16400, 16400, "public", "t")); + catalog.insert(desc(16402, 16402, "pg_toast", "pg_toast_16400")); + + let (filter, targets) = replay_scope(&catalog); + + assert_eq!(filter.len(), 2); + assert_eq!( + targets.keys().map(|(_, rel)| *rel).collect::>(), + [16400] + ); + assert!(targets.values().all(|(_, ceiling)| *ceiling == u64::MAX)); + } + + /// End on a boundary stops at the segment holding the last byte + #[tokio::test] + async fn segments_in_dir_stops_below_a_boundary_end() { + let tmp = tempfile::tempdir().unwrap(); + let names: Vec = (1..=2) + .map(|n| { + SegmentName { + timeline: 1, + log_id: 0, + seg_no: n, + } + .format() + }) + .collect(); + for n in &names { + tokio::fs::write(tmp.path().join(n), b"").await.unwrap(); + } + let segs = segments_in_dir(tmp.path(), 1, WAL_SEG_SIZE + 1024, 3 * WAL_SEG_SIZE) + .await + .unwrap(); + assert_eq!( + segs.iter().map(|(s, _)| s.format()).collect::>(), + names, + ); + } + + /// Reject incomplete segment ranges + #[tokio::test] + async fn segments_in_dir_spans_the_window_and_refuses_gaps() { + let tmp = tempfile::tempdir().unwrap(); + let from = WAL_SEG_SIZE + 1024; + let end = 3 * WAL_SEG_SIZE + 512; + let names: Vec = (1..=3) + .map(|n| { + SegmentName { + timeline: 1, + log_id: 0, + seg_no: n, + } + .format() + }) + .collect(); + for n in &names[..2] { + tokio::fs::write(tmp.path().join(n), b"").await.unwrap(); + } + let err = segments_in_dir(tmp.path(), 1, from, end) + .await + .expect_err("third segment missing"); + assert!(err.to_string().contains(&names[2]), "{err}"); + + tokio::fs::write(tmp.path().join(&names[2]), b"") + .await + .unwrap(); + let segs = segments_in_dir(tmp.path(), 1, from, end).await.unwrap(); + assert_eq!( + segs.iter().map(|(s, _)| s.format()).collect::>(), + names, + ); + } +} diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 1aafc293..0de4c761 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -71,7 +71,8 @@ use crate::config::ResolvedConfig; use crate::decode::codecs::NumericKind; use crate::decode::heap_decoder::ColumnValue; use crate::emit::ch_emitter::{EmitterConfig, EmitterStats}; -use crate::emit::pipeline::{Fatal, bootstrap, tail}; +use crate::emit::pipeline::tail::OwnedTail; +use crate::emit::pipeline::{Fatal, bootstrap}; use crate::mapping::MappingHandle; use crate::ops::oracle::Oracle; use crate::pg::{current_wal_lsn, quote_ident}; @@ -298,7 +299,7 @@ fn wire_kind(type_oid: u32) -> Option { fn column_plan(desc: &RelDescriptor) -> CopyPlan { let mut select = String::new(); - let mut cols = Vec::new(); + let mut cols = Vec::with_capacity(desc.attributes.len()); let mut natts = 0usize; for a in &desc.attributes { natts = natts.max(a.attnum.max(0) as usize); @@ -328,6 +329,55 @@ fn column_plan(desc: &RelDescriptor) -> CopyPlan { } } +/// Copy visible, detoasted rows from `desc` into `tx`, tagged `lsn` +pub(crate) async fn copy_rows_into( + client: &tokio_postgres::Client, + desc: &RelDescriptor, + lsn: u64, + tx: &mpsc::Sender, +) -> anyhow::Result { + let plan = column_plan(desc); + let sql = format!( + "COPY (SELECT {} FROM ONLY {}.{}) TO STDOUT (FORMAT binary)", + plan.select, + quote_ident(&desc.rel_name.namespace), + quote_ident(&desc.rel_name.name), + ); + let byte_fields = vec![Type::BYTEA; plan.cols.len()]; + let copy = client.copy_out(&sql).await.context("backfill: COPY out")?; + let stream = BinaryCopyOutStream::new(copy, &byte_fields); + futures::pin_mut!(stream); + let mut rows = 0u64; + while let Some(row) = stream.next().await { + let row = row.context("backfill: COPY stream")?; + let mut columns: Vec> = vec![None; plan.natts]; + for (i, cp) in plan.cols.iter().enumerate() { + let raw: Option<&[u8]> = row.try_get(i).context("backfill: COPY field")?; + let v = raw + .map(|raw| decode_field(cp.kind, cp.type_oid, raw)) + .transpose() + .map_err(anyhow::Error::msg)? + .unwrap_or(ColumnValue::Null); + columns[(cp.attnum - 1).max(0) as usize] = Some(v); + } + tx.send(BackfillTuple { + rfn: desc.rfn, + xid: 0, + xmax: 0, + infomask: 0, + source_lsn: lsn, + // COPY rows have no on-page TID + blkno: 0, + offnum: 0, + columns, + }) + .await + .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; + rows += 1; + } + Ok(rows) +} + fn fixed(raw: &[u8], what: &str) -> Result<[u8; N], String> { raw.try_into() .map_err(|_| format!("{what}: expected {N} bytes, got {}", raw.len())) @@ -777,7 +827,7 @@ impl CopyBackfiller { return; } }; - let mut swapped: Vec<&StagingRel> = Vec::new(); + let mut swapped: Vec<&StagingRel> = Vec::with_capacity(plan.rels.len()); for rel in &plan.rels { match self.swap_rel(&mut sess, rel).await { Ok(true) => swapped.push(rel), @@ -1039,10 +1089,14 @@ impl CopyBackfiller { let client = open_sql_client(&self.source_pg()) .await .context("backfill: source sql connect")?; + client + .batch_execute("SET row_security = off") + .await + .context("backfill: reject row security filtering")?; // Empty table ⇒ streaming alone suffices, skip COPY + tail entirely let nonempty: bool = client - .query_one(&format!("SELECT EXISTS (SELECT 1 FROM {qtable})"), &[]) + .query_one(&format!("SELECT EXISTS (SELECT 1 FROM ONLY {qtable})"), &[]) .await .context("backfill: emptiness probe")? .get(0); @@ -1056,18 +1110,17 @@ impl CopyBackfiller { } // Dedicated tail: own CH connection, own seq space, own fatal. - let fatal = Fatal::new(); - let (msg_tx, ack, tail) = tail::spawn_with_config( + let tail = OwnedTail::spawn( &self.dest_emitter(), 1, self.stats.clone(), - Arc::new(crate::pos::Monotone::new(0)), - fatal.clone(), + Fatal::new(), self.config_rx.clone(), self.oracle.clone(), + "backfill", ) .await - .map_err(|e| anyhow::anyhow!("backfill: spawn insert tail: {e}"))?; + .map_err(anyhow::Error::msg)?; let mut catalog = CatalogMap::new(); catalog.insert(desc.clone()); @@ -1076,8 +1129,8 @@ impl CopyBackfiller { tup_rx, catalog, self.mapping.clone(), - msg_tx.clone(), - ack.clone(), + tail.msg_tx.clone(), + tail.ack.clone(), self.stats.clone(), // Disabled resolver never defers; spool stays empty ToastResolver::disabled(), @@ -1087,51 +1140,10 @@ impl CopyBackfiller { ), self.emitter.row_policy(), self.config_rx.as_ref().map(|rx| rx.borrow().clone()), - std::collections::HashSet::new(), + HashSet::new(), )); - let plan = column_plan(desc); - let sql = format!( - "COPY (SELECT {} FROM {qtable}) TO STDOUT (FORMAT binary)", - plan.select - ); - let rows = { - let byte_fields = vec![Type::BYTEA; plan.cols.len()]; - let copy = client.copy_out(&sql).await.context("backfill: COPY out")?; - let stream = BinaryCopyOutStream::new(copy, &byte_fields); - futures::pin_mut!(stream); - let mut rows = 0u64; - while let Some(row) = stream.next().await { - let row = row.context("backfill: COPY stream")?; - let mut columns: Vec> = vec![None; plan.natts]; - for (i, cp) in plan.cols.iter().enumerate() { - let raw: Option<&[u8]> = row.try_get(i).context("backfill: COPY field")?; - let v = raw - .map(|raw| decode_field(cp.kind, cp.type_oid, raw)) - .transpose() - .map_err(anyhow::Error::msg)? - .unwrap_or(ColumnValue::Null); - columns[(cp.attnum - 1).max(0) as usize] = Some(v); - } - tup_tx - .send(BackfillTuple { - rfn: desc.rfn, - xid: 0, - xmax: 0, - infomask: 0, - source_lsn: s_lsn.get(), - // COPY text rows have no on-page TID (values arrive - // detoasted, no chunks flow here) - blkno: 0, - offnum: 0, - columns, - }) - .await - .map_err(|_| anyhow::anyhow!("backfill: drain closed early"))?; - rows += 1; - } - rows - }; + let rows = copy_rows_into(&client, desc, s_lsn.get(), &tup_tx).await?; drop(tup_tx); let outcome = drain @@ -1140,7 +1152,7 @@ impl CopyBackfiller { .map_err(anyhow::Error::msg)?; // Upper bound on the COPY snapshot; WAL apply past it = converged let p_hi = current_wal_lsn(&client).await?; - tail.finish(msg_tx, ack, outcome.next_seq, &fatal) + tail.finish(outcome.next_seq) .await .map_err(anyhow::Error::msg)?; Ok(CopyOutcome { diff --git a/src/backfill/mod.rs b/src/backfill/mod.rs index 890dafca..f0c5a9a9 100644 --- a/src/backfill/mod.rs +++ b/src/backfill/mod.rs @@ -9,7 +9,11 @@ pub mod backup_source; pub mod backup_source_direct; pub mod backup_source_object_store; pub mod bootstrap_oracle; +pub mod bootstrap_window; pub mod copy_backfill; pub mod opt_in; pub mod pg_path; pub mod spool; +pub mod visibility_gate; +pub mod visibility_repair; +pub mod wal_replay; diff --git a/src/backfill/visibility_gate.rs b/src/backfill/visibility_gate.rs new file mode 100644 index 00000000..2f71baeb --- /dev/null +++ b/src/backfill/visibility_gate.rs @@ -0,0 +1,653 @@ +//! Filter backup-page tuples by PostgreSQL visibility +//! +//! [`stream_phase`] resolves hint bits and spools unknowns. [`resolve_phase`] +//! uses complete backup transaction logs plus WAL commit overlay. Greenfield +//! repairs relations whose tuples remain undecidable + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use tokio::sync::mpsc; +use walrus::pg::replication::conn::PgConfig; + +use crate::backfill::backup_page_walk::{BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTuple, CatalogMap}; +use crate::backfill::spool::{DEFERRED_SPOOL_MEM_MAX, DeferredSpool}; +use crate::backfill::visibility_repair::{PendingReason, PendingSet, RepairStats, repair}; +use crate::config::ResolvedConfig; +use crate::decode::visibility::{ + HEAP_XMAX_IS_MULTI, PgMultiXactAccum, PgXactAccum, PgXactPatch, PgXactView, Visibility, + read_pg_multixact, read_pg_xact, tuple_visibility, +}; +use crate::emit::ch_emitter::{EmitterConfig, EmitterStats}; +use crate::emit::pipeline::tail::OwnedTail; +use crate::emit::pipeline::{Fatal, bootstrap}; +use crate::mapping::MappingHandle; +use crate::schema::RelName; +use crate::toast::ToastResolver; +use ahash::HashSet; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct GateStats { + pub emitted: u64, + pub gated: u64, + pub deferred: u64, + pub multixact_emitted: u64, + /// Chunk tuples the hint bits proved dead, dropped before the store + pub chunks_gated: u64, + /// Walked tuples replaced by relation repair + pub pending_discarded: u64, + /// Relations handed to visibility repair + pub pending_relations: u64, + /// Relations pending for undecidable multixacts + pub pending_multixact: u64, + /// Rows emitted by visibility repair + pub repaired_rows: u64, + /// Source write head after final repair read + pub p_hi: u64, +} + +/// Policy for tuples remaining undecidable +#[derive(Debug)] +pub enum Undecidable<'a> { + /// Abort per-table loads + Abort, + /// Hand greenfield relation to repair + Pending(&'a mut PendingSet), +} + +/// Resolve hint bits, spool unknown main tuples, pass chunks unless proven dead +pub async fn stream_phase( + rx: &mut mpsc::Receiver, + tx: &mpsc::Sender, + catalog: &CatalogMap, + pending: &mut PendingSet, + deferred: &mut DeferredSpool, + stats: &mut GateStats, +) -> Result<(), String> { + while let Some(t) = rx.recv().await { + if catalog.is_toast(t.rfn.db_node, t.rfn.rel_node) { + if tuple_visibility(t.xid, t.xmax, t.infomask, None) == Visibility::Skip { + stats.chunks_gated += 1; + continue; + } + if tx.send(t).await.is_err() { + break; + } + continue; + } + // Repair replaces main-page tuples + if pending.holds(t.rfn.db_node, t.rfn.rel_node) { + stats.pending_discarded += 1; + continue; + } + let visibility = tuple_visibility(t.xid, t.xmax, t.infomask, None); + if visibility != Visibility::Skip && pending.observe_external(&t) { + stats.pending_discarded += 1; + continue; + } + match visibility { + Visibility::Emit => { + if !emit(t, tx, stats).await { + break; + } + } + Visibility::Skip => stats.gated += 1, + // Multixact verdict requires complete view + Visibility::Defer | Visibility::Unresolvable => deferred + .push(t) + .await + .map_err(|e| format!("visibility gate: deferred spool: {e}"))?, + } + } + stats.deferred = deferred.records(); + Ok(()) +} + +/// Replay spool against complete transaction view +pub async fn resolve_phase( + deferred: DeferredSpool, + view: &PgXactView<'_>, + tx: &mpsc::Sender, + mut undecidable: Undecidable<'_>, + stats: &mut GateStats, +) -> Result<(), String> { + let mut replay = deferred + .into_reader() + .await + .map_err(|e| format!("visibility gate: deferred spool seal: {e}"))?; + let mut fail = None; + while let Some(t) = replay + .next() + .await + .map_err(|e| format!("visibility gate: deferred spool replay: {e}"))? + { + if let Undecidable::Pending(set) = &undecidable + && set.holds(t.rfn.db_node, t.rfn.rel_node) + { + stats.pending_discarded += 1; + continue; + } + match tuple_visibility(t.xid, t.xmax, t.infomask, Some(view)) { + Visibility::Emit => { + if !emit(t, tx, stats).await { + break; + } + } + Visibility::Skip | Visibility::Defer => stats.gated += 1, + Visibility::Unresolvable => match &mut undecidable { + Undecidable::Abort => { + fail = Some(undecidable_multixact(&t)); + break; + } + Undecidable::Pending(set) => { + set.mark(t.rfn, PendingReason::UnresolvedMultiXact); + stats.pending_discarded += 1; + } + }, + } + } + replay + .finish() + .await + .map_err(|e| format!("visibility gate: deferred spool cleanup: {e}"))?; + match fail { + Some(e) => Err(e), + None => Ok(()), + } +} + +async fn emit(t: BackfillTuple, tx: &mpsc::Sender, stats: &mut GateStats) -> bool { + if t.infomask & HEAP_XMAX_IS_MULTI != 0 { + stats.multixact_emitted += 1; + } + stats.emitted += 1; + tx.send(t).await.is_ok() +} + +fn undecidable_multixact(t: &BackfillTuple) -> String { + format!( + "visibility gate: multixact xmax {} (rfn {}/{}) unresolvable from the backup's \ + pg_multixact snapshot; remedy: fresher backup, or initial_load='copy'", + t.xmax, t.rfn.db_node, t.rfn.rel_node + ) +} + +/// Greenfield resolution inputs +pub struct PendingGate { + pub deferred: DeferredSpool, + pub catalog: CatalogMap, + pub mapping: MappingHandle, + pub config: Arc, + pub emitter: EmitterConfig, + pub stats: Arc, + pub resolver: ToastResolver, + pub oracle: Option>, + /// Relations excluded from initial load + pub skip_initial: HashSet, + /// Source endpoint for repair reads + pub source: PgConfig, + /// Relations handed to repair + pub pending: PendingSet, + /// Coverage tag for walked and repaired rows + pub start_lsn: u64, + /// Repair spill root + pub spill_dir: PathBuf, + /// Streaming-phase counters + pub stream_stats: GateStats, +} + +/// Resolve deferred tuples and repair pending relations +pub async fn resolve_greenfield( + gate: PendingGate, + data_dir: &Path, + patch: &PgXactPatch, +) -> Result { + let PendingGate { + deferred, + catalog, + mapping, + config, + emitter, + stats, + resolver, + oracle, + skip_initial, + source, + mut pending, + start_lsn, + spill_dir, + stream_stats: mut gate_stats, + } = gate; + if deferred.records() == 0 && pending.is_empty() { + return Ok(gate_stats); + } + // Repair rides the same drain, so it needs its own view of what the + // drain takes ownership of + let repair_catalog = catalog.clone(); + let repair_skip = skip_initial.clone(); + // SLRUs only answer the spool; a pending-only pass reads none of them + let (accum, multi) = if deferred.records() == 0 { + (PgXactAccum::new(), PgMultiXactAccum::new()) + } else { + ( + read_pg_xact(data_dir).await?, + read_pg_multixact(data_dir).await?, + ) + }; + let view = PgXactView::new(&accum, patch).with_multixact(&multi); + + let tail = OwnedTail::spawn( + &emitter, + 1, + stats.clone(), + Fatal::new(), + None, + oracle, + "visibility gate", + ) + .await + .map_err(anyhow::Error::msg)?; + + let toast_spool = spill_dir.join("bootstrap_gate_toast.bin"); + tokio::fs::remove_file(&toast_spool).await.ok(); + let (tx, rx) = mpsc::channel::(BOOTSTRAP_TUPLE_CHANNEL_CAP); + let drain = tokio::spawn(bootstrap::drain( + rx, + catalog, + mapping, + tail.msg_tx.clone(), + tail.ack.clone(), + stats, + resolver, + DeferredSpool::new(toast_spool, DEFERRED_SPOOL_MEM_MAX), + emitter.row_policy(), + Some(config), + skip_initial, + )); + + let mut resolved = resolve_phase( + deferred, + &view, + &tx, + Undecidable::Pending(&mut pending), + &mut gate_stats, + ) + .await; + gate_stats.pending_relations = pending.len() as u64; + gate_stats.pending_multixact = pending.count_for(PendingReason::UnresolvedMultiXact); + if resolved.is_ok() { + match repair( + &pending, + &repair_catalog, + &repair_skip, + &source, + start_lsn, + &tx, + ) + .await + { + Ok(r) => { + gate_stats.repaired_rows = r.rows; + gate_stats.p_hi = r.p_hi; + log_repair(&r, gate_stats.pending_multixact); + } + Err(e) => resolved = Err(format!("{e:#}")), + } + } + drop(tx); + let drained = drain.await.context("visibility gate: drain join")?; + // Drain tail before surfacing errors + let next_seq = match (resolved, drained) { + (Ok(()), Ok(o)) => o.next_seq, + (Err(e), _) | (_, Err(e)) => { + tail.quiesce().await; + anyhow::bail!(e); + } + }; + tail.finish(next_seq).await.map_err(anyhow::Error::msg)?; + Ok(gate_stats) +} + +/// Log repair convergence frontier +fn log_repair(r: &RepairStats, multixact_relations: u64) { + if r.relations == 0 && r.skipped == 0 { + return; + } + tracing::info!( + target: "walshadow::bootstrap", + relations = r.relations, + rows = r.rows, + skipped = r.skipped, + multixact_relations, + p_hi = r.p_hi, + "visibility repair read pending relations through PostgreSQL", + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backfill::backup_page_walk::make_rel_named; + use crate::decode::visibility::{ + HEAP_XMAX_COMMITTED, HEAP_XMAX_INVALID, HEAP_XMAX_IS_MULTI, HEAP_XMIN_COMMITTED, + HEAP_XMIN_INVALID, + }; + use ahash::HashSetExt; + use walrus::pg::walparser::RelFileNode; + + fn rfn(rel_node: u32) -> RelFileNode { + RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node, + } + } + + fn tuple(xid: u32, xmax: u32, infomask: u16) -> BackfillTuple { + BackfillTuple { + rfn: rfn(16400), + xid, + xmax, + infomask, + source_lsn: 0x1000, + blkno: 0, + offnum: 0, + columns: Vec::new(), + } + } + + async fn spool_of(tuples: Vec) -> DeferredSpool { + let mut spool = DeferredSpool::new( + std::env::temp_dir().join("ws-visibility-gate-unused.bin"), + DEFERRED_SPOOL_MEM_MAX, + ); + for t in tuples { + spool.push(t).await.unwrap(); + } + spool + } + + fn toast_catalog(rel_node: u32) -> CatalogMap { + let mut catalog = CatalogMap::new(); + catalog.insert(make_rel_named( + rel_node, + rel_node, + 0, + RelName::new("pg_toast", &format!("pg_toast_{rel_node}")), + )); + catalog + } + + /// Drive the walk phase over `tuples`, return its tally and whatever it + /// let through + async fn run_stream_phase( + catalog: &CatalogMap, + pending: &mut PendingSet, + tuples: Vec, + ) -> (GateStats, Vec) { + let (tx, mut rx) = mpsc::channel(8); + let (walk_tx, mut walk_rx) = mpsc::channel(8); + for t in tuples { + walk_tx.send(t).await.unwrap(); + } + drop(walk_tx); + + let mut stats = GateStats::default(); + let mut spool = spool_of(Vec::new()).await; + stream_phase(&mut walk_rx, &tx, catalog, pending, &mut spool, &mut stats) + .await + .unwrap(); + drop(tx); + + let mut passed = Vec::new(); + while let Some(t) = rx.recv().await { + passed.push(t); + } + (stats, passed) + } + + #[tokio::test] + async fn repair_requires_a_mapped_external_value() { + use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; + use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; + use crate::schema::RelName; + + let mut desc = make_rel_named(16400, 16400, 16402, RelName::new("public", "t")); + let d = Arc::make_mut(&mut desc); + let mut body = d.attributes[0].clone(); + body.attnum = 2; + body.name = "body".into(); + body.type_oid = crate::schema::TEXTOID; + body.type_len = -1; + // SET STORAGE PLAIN leaves existing external values untouched + body.type_storage = 'p'; + d.attributes.push(body); + let mut catalog = CatalogMap::new(); + catalog.insert(desc.clone()); + let external = ColumnValue::ExternalToast(ToastPointer { + va_rawsize: 8004, + va_extinfo: 8000, + va_valueid: 100, + va_toastrelid: 16402, + }); + for body_mapped in [false, true] { + let mut columns = vec![ColumnMapping { + src_attnum: 1, + target_name: "id".into(), + target_type: "Int32".into(), + }]; + if body_mapped { + columns.push(ColumnMapping { + src_attnum: 2, + target_name: "body".into(), + target_type: "String".into(), + }); + } + let mapping = [( + desc.rel_name.clone(), + TableMapping { + target: TableTarget::new("default", "t"), + columns, + }, + )] + .into_iter() + .collect::>() + .into(); + let mut pending = PendingSet::mapped(&catalog, &mapping); + assert!(pending.is_empty()); + let row = |value, infomask| BackfillTuple { + columns: vec![Some(ColumnValue::Int4(1)), Some(value)], + ..tuple(100, 0, infomask) + }; + let (stats, passed) = run_stream_phase( + &catalog, + &mut pending, + vec![ + row(ColumnValue::Text("inline".into()), HEAP_XMIN_COMMITTED), + row(external.clone(), HEAP_XMIN_INVALID), + row(external.clone(), HEAP_XMIN_COMMITTED), + ], + ) + .await; + assert_eq!(stats.gated, 1); + assert_eq!(pending.holds(5, 16400), body_mapped); + assert_eq!(passed.len(), if body_mapped { 1 } else { 2 }); + } + } + + /// Drop proven-dead chunk generations + #[tokio::test] + async fn proven_dead_chunks_never_reach_the_store() { + let chunk = |xmax, infomask| BackfillTuple { + rfn: rfn(16401), + xmax, + infomask, + ..tuple(100, 0, 0) + }; + // Deleted value, aborted insert, then one that is still live + let (stats, passed) = run_stream_phase( + &toast_catalog(16401), + &mut PendingSet::empty(), + vec![ + chunk(200, HEAP_XMIN_COMMITTED | HEAP_XMAX_COMMITTED), + chunk(0, HEAP_XMIN_INVALID), + chunk(0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID), + ], + ) + .await; + + assert_eq!(stats.chunks_gated, 2); + assert_eq!(passed.len(), 1, "only the live chunk passes"); + // Chunks never take the main-tuple counters or the spool + assert_eq!(stats.emitted, 0); + assert_eq!(stats.deferred, 0); + } + + /// Repair replaces main rows but preserves chunk cache + #[tokio::test] + async fn pending_relation_drops_its_pages_and_keeps_its_chunks() { + let mut catalog = CatalogMap::new(); + catalog.insert(make_rel_named( + 16400, + 16400, + 16402, + RelName::new("public", "t"), + )); + catalog.insert(make_rel_named( + 16402, + 16403, + 0, + RelName::new("pg_toast", "pg_toast_16400"), + )); + let mut pending = PendingSet::toast_capable(&catalog); + + let live = HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID; + let (stats, passed) = run_stream_phase( + &catalog, + &mut pending, + vec![ + tuple(100, 0, live), + BackfillTuple { + rfn: rfn(16403), + ..tuple(100, 0, live) + }, + ], + ) + .await; + + assert_eq!(stats.pending_discarded, 1, "its main page is dropped"); + assert_eq!(stats.emitted, 0); + assert_eq!( + passed.iter().map(|t| t.rfn.rel_node).collect::>(), + [16403], + "only its chunk reaches the store", + ); + } + + /// Preserve chunks without decisive hint bits + #[tokio::test] + async fn undecidable_chunks_still_pass_through() { + let (stats, passed) = run_stream_phase( + &toast_catalog(16401), + &mut PendingSet::empty(), + vec![BackfillTuple { + rfn: rfn(16401), + ..tuple(100, 0, 0) + }], + ) + .await; + + assert_eq!(stats.chunks_gated, 0); + assert_eq!(passed.len(), 1); + assert_eq!(stats.deferred, 0, "chunks never spool"); + } + + #[tokio::test] + async fn greenfield_hands_undecidable_multixact_relation_over() { + let accum = PgXactAccum::new(); + let patch = PgXactPatch::new(); + // No collected offsets segment: the mxid is below any known range + let multi = PgMultiXactAccum::new(); + let view = PgXactView::new(&accum, &patch).with_multixact(&multi); + let (tx, mut rx) = mpsc::channel(4); + let mut stats = GateStats::default(); + let spool = spool_of(vec![ + tuple(100, 10, HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI), + tuple(100, 0, HEAP_XMIN_COMMITTED), + ]) + .await; + + let mut pending = PendingSet::empty(); + resolve_phase( + spool, + &view, + &tx, + Undecidable::Pending(&mut pending), + &mut stats, + ) + .await + .unwrap(); + drop(tx); + + assert!( + rx.recv().await.is_none(), + "an undecidable tuple is never guessed" + ); + assert_eq!(stats.emitted, 0); + assert_eq!(stats.pending_discarded, 2); + assert_eq!(pending.len(), 1, "its relation goes to the repair path"); + assert_eq!(pending.count_for(PendingReason::UnresolvedMultiXact), 1); + assert!(pending.holds(5, 16400)); + } + + #[tokio::test] + async fn per_table_aborts_on_undecidable_multixact() { + let accum = PgXactAccum::new(); + let patch = PgXactPatch::new(); + let multi = PgMultiXactAccum::new(); + let view = PgXactView::new(&accum, &patch).with_multixact(&multi); + let (tx, _rx) = mpsc::channel(4); + let mut stats = GateStats::default(); + let spool = spool_of(vec![tuple( + 100, + 10, + HEAP_XMIN_COMMITTED | HEAP_XMAX_IS_MULTI, + )]) + .await; + + let err = resolve_phase(spool, &view, &tx, Undecidable::Abort, &mut stats) + .await + .unwrap_err(); + assert!(err.contains("pg_multixact"), "{err}"); + assert_eq!(stats.emitted, 0); + } + + /// Nothing deferred: no tail, no CH connection, no work + #[tokio::test] + async fn resolve_greenfield_is_a_noop_without_deferrals() { + let pending = PendingGate { + deferred: spool_of(Vec::new()).await, + catalog: CatalogMap::new(), + mapping: crate::mapping::mapping_handle(Default::default()), + config: Arc::new(ResolvedConfig::default()), + emitter: EmitterConfig::default(), + stats: Arc::new(EmitterStats::default()), + resolver: ToastResolver::disabled(), + oracle: None, + skip_initial: HashSet::new(), + source: crate::config::SourceConn::default().to_pg_config(), + pending: PendingSet::empty(), + start_lsn: 0x1000, + spill_dir: std::env::temp_dir(), + stream_stats: GateStats { + emitted: 7, + ..Default::default() + }, + }; + let stats = resolve_greenfield(pending, Path::new("/nonexistent"), &PgXactPatch::new()) + .await + .unwrap(); + assert_eq!(stats.emitted, 7); + } +} diff --git a/src/backfill/visibility_repair.rs b/src/backfill/visibility_repair.rs new file mode 100644 index 00000000..28d95193 --- /dev/null +++ b/src/backfill/visibility_repair.rs @@ -0,0 +1,341 @@ +//! Replace relations whose backup-page visibility cannot be proven +//! +//! Read each pending relation through PostgreSQL `COPY`, emit visible, +//! detoasted rows at page-walk coverage LSN. Keep chunk pages in mirror for +//! later WAL rows carrying old external pointers + +use anyhow::{Context, Result, bail}; +use tokio::sync::mpsc; +use walrus::pg::replication::conn::PgConfig; +use walrus::pg::walparser::{Oid, RelFileNode}; + +use crate::backfill::backfill_bootstrap::seed_in_snapshot; +use crate::backfill::backup_page_walk::{BackfillTuple, CatalogMap}; +use crate::backfill::copy_backfill::copy_rows_into; +use crate::decode::heap_decoder::ColumnValue; +use crate::mapping::MappingSnapshot; +use crate::pg::current_wal_lsn; +use crate::schema::{RelDescriptor, RelName}; +use crate::source::source_feed::open_sql_client; +use ahash::{HashMap, HashSet}; + +/// Reason for authoritative relation read +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PendingReason { + /// `xmax` multixact outside backup coverage + UnresolvedMultiXact, + /// Mapped external value may reference ambiguous chunk generations + ExternalToast, +} + +/// Relations handed from page walk to repair +#[derive(Debug, Default)] +pub struct PendingSet { + rels: HashMap<(Oid, Oid), PendingReason>, + /// Filenodes repair may read, `None` unrestricted. Greenfield scopes to + /// the mapping snapshot: the drain routes by mapping, so reading an + /// unmapped relation ships rows it drops + scope: Option>, + external: HashMap<(Oid, Oid), Vec>, +} + +impl PendingSet { + /// Empty set for per-table loads without repair + pub fn empty() -> Self { + Self::default() + } + + pub fn mapped(catalog: &CatalogMap, mapping: &MappingSnapshot) -> Self { + let mut set = Self::empty(); + let mut scope = HashSet::default(); + for desc in catalog.descriptors() { + let Some(mapped) = mapping.get(&desc.rel_name) else { + continue; + }; + let key = (desc.rfn.db_node, desc.rfn.rel_node); + scope.insert(key); + if desc.toast_oid == 0 { + continue; + } + let columns: Vec<_> = mapped + .columns + .iter() + .filter_map(|c| { + let attr = desc.attributes.iter().find(|a| a.attnum == c.src_attnum)?; + (attr.type_len == -1 && !attr.dropped) + .then(|| usize::try_from(i32::from(attr.attnum) - 1).ok()) + .flatten() + }) + .collect(); + if !columns.is_empty() { + set.external.insert(key, columns); + } + } + set.scope = Some(scope); + set + } + + pub fn observe_external(&mut self, tuple: &BackfillTuple) -> bool { + let key = (tuple.rfn.db_node, tuple.rfn.rel_node); + let external = self.external.get(&key).is_some_and(|columns| { + columns.iter().any(|&i| { + matches!( + tuple.columns.get(i), + Some(Some(ColumnValue::ExternalToast(_))) + ) + }) + }); + if external { + self.mark(tuple.rfn, PendingReason::ExternalToast); + } + external + } + + /// Pre-mark relations owning TOAST storage + pub fn toast_capable(catalog: &CatalogMap) -> Self { + let mut set = Self::default(); + for desc in catalog.descriptors().filter(|d| d.toast_oid != 0) { + set.rels.insert( + (desc.rfn.db_node, desc.rfn.rel_node), + PendingReason::ExternalToast, + ); + } + set + } + + /// Restrict repair to `mapped` filenodes, dropping pre-marks outside it + pub fn scoped_to(mut self, mapped: HashSet<(Oid, Oid)>) -> Self { + self.rels.retain(|k, _| mapped.contains(k)); + self.external.retain(|k, _| mapped.contains(k)); + self.scope = Some(mapped); + self + } + + /// Hand a relation over mid-walk. First reason wins. Out of scope is + /// dropped: the tuple was headed for the drain's discard either way + pub fn mark(&mut self, rfn: RelFileNode, reason: PendingReason) { + let key = (rfn.db_node, rfn.rel_node); + if self.scope.as_ref().is_some_and(|s| !s.contains(&key)) { + return; + } + self.rels.entry(key).or_insert(reason); + } + + /// Check whether repair replaces relation main pages + pub fn holds(&self, db_node: Oid, rel_node: Oid) -> bool { + self.rels.contains_key(&(db_node, rel_node)) + } + + pub fn is_empty(&self) -> bool { + self.rels.is_empty() + } + + pub fn len(&self) -> usize { + self.rels.len() + } + + pub fn count_for(&self, reason: PendingReason) -> u64 { + self.rels.values().filter(|r| **r == reason).count() as u64 + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct RepairStats { + pub relations: u64, + pub rows: u64, + /// Relations `initial_load = "none"` opted out of + pub skipped: u64, + /// Source write head after final read + pub p_hi: u64, +} + +/// Read pending relations through PostgreSQL at `coverage_lsn` +/// +/// Reject descriptors changed since page walk +pub async fn repair( + pending: &PendingSet, + catalog: &CatalogMap, + skip_initial: &HashSet, + source: &PgConfig, + coverage_lsn: u64, + tx: &mpsc::Sender, +) -> Result { + let mut stats = RepairStats::default(); + if pending.is_empty() { + return Ok(stats); + } + let client = open_sql_client(source) + .await + .context("visibility repair: source sql connect")?; + client + .batch_execute("SET row_security = off") + .await + .context("visibility repair: reject row security filtering")?; + // Re-read the same seed the walk's catalog came from, so an unchanged + // relation compares equal field for field + let fresh = seed_in_snapshot(&client) + .await + .context("visibility repair: re-seed source catalog")?; + + for &(db_node, rel_node) in pending.rels.keys() { + let desc = catalog.get(db_node, rel_node).with_context(|| { + format!("visibility repair: filenode {db_node}/{rel_node} left the catalog map") + })?; + if skip_initial.contains(&desc.rel_name) { + stats.skipped += 1; + continue; + } + assert_unchanged(&fresh, &desc)?; + stats.relations += 1; + stats.rows += copy_rows_into(&client, &desc, coverage_lsn, tx) + .await + .with_context(|| format!("visibility repair: COPY {}", desc.rel_name))?; + } + // Sample after reads to bound every snapshot + stats.p_hi = current_wal_lsn(&client) + .await + .context("visibility repair: source write head")?; + Ok(stats) +} + +/// Verify COPY target still matches walked descriptor +/// +/// Coarse by design: bootstrap does not support DDL inside the backup +/// window, so any drift ends the pass +fn assert_unchanged(fresh: &CatalogMap, desc: &RelDescriptor) -> Result<()> { + match fresh.get(desc.rfn.db_node, desc.rfn.rel_node) { + Some(now) if *now == *desc => Ok(()), + Some(now) => bail!( + "visibility repair: relation {} changed inside the backup window \ + ({} to {}); rerun bootstrap against a quiesced source", + desc.rel_name, + shape(desc), + shape(&now), + ), + // Filenode is the map key, so a rewrite moved the oid elsewhere + None => match fresh.descriptors().find(|d| d.oid == desc.oid) { + Some(moved) => bail!( + "visibility repair: relation {} was rewritten inside the backup window \ + (filenode {} to {}); rerun bootstrap against a quiesced source", + desc.rel_name, + desc.rfn.rel_node, + moved.rfn.rel_node, + ), + None => bail!( + "visibility repair: relation {} (oid {}) is gone from the source; \ + bootstrap does not support DDL inside the backup window", + desc.rel_name, + desc.oid, + ), + }, + } +} + +/// Descriptor shape drift reports name +fn shape(d: &RelDescriptor) -> String { + let cols: Vec<&str> = d + .attributes + .iter() + .filter(|a| !a.dropped) + .map(|a| a.name.as_str()) + .collect(); + format!( + "{}/{}/{} [{}]", + d.rel_name, + d.kind, + d.replident.to_char(), + cols.join(","), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backfill::backup_page_walk::make_rel_named; + use ahash::HashSetExt; + use std::sync::Arc; + + fn desc( + oid: Oid, + rel_node: Oid, + toast_oid: Oid, + namespace: &str, + name: &str, + ) -> Arc { + make_rel_named(oid, rel_node, toast_oid, RelName::new(namespace, name)) + } + + /// A relation with a toast relation is pending before the walk. Its + /// chunk store is not: the mirror is a cache the walk keeps filling + #[test] + fn toast_capable_scope_takes_the_parent_but_not_its_chunks() { + let mut catalog = CatalogMap::new(); + catalog.insert(desc(16400, 16400, 16402, "public", "with_text")); + catalog.insert(desc(16402, 16403, 0, "pg_toast", "pg_toast_16400")); + catalog.insert(desc(16500, 16500, 0, "public", "fixed_width")); + + let set = PendingSet::toast_capable(&catalog); + + assert_eq!(set.len(), 1); + assert!(set.holds(5, 16400), "parent is pending"); + assert!(!set.holds(5, 16403), "its chunk filenode still walks"); + assert!(!set.holds(5, 16500), "a fixed-width relation still walks"); + assert_eq!(set.count_for(PendingReason::ExternalToast), 1); + } + + /// Repair reads the source, so scope follows the drain's routes: an + /// unmapped relation's rows would be dropped on arrival + #[test] + fn scope_drops_unmapped_relations_and_their_later_marks() { + let mut catalog = CatalogMap::new(); + catalog.insert(desc(16400, 16400, 16402, "public", "mapped")); + catalog.insert(desc(16500, 16500, 16502, "public", "unmapped")); + + let mut set = + PendingSet::toast_capable(&catalog).scoped_to([(5, 16400)].into_iter().collect()); + + assert_eq!(set.len(), 1); + assert!(set.holds(5, 16400)); + assert!(!set.holds(5, 16500), "pre-mark outside the mapping drops"); + + let unmapped = catalog.get(5, 16500).unwrap(); + set.mark(unmapped.rfn, PendingReason::UnresolvedMultiXact); + assert!(!set.holds(5, 16500), "so does a walk-time mark"); + } + + /// Marking mid-walk keeps the first reason + #[test] + fn walk_time_mark_keeps_the_first_reason() { + let mut catalog = CatalogMap::new(); + catalog.insert(desc(16400, 16400, 16402, "public", "t")); + let mut set = PendingSet::empty(); + assert!(!set.holds(5, 16400)); + + let parent = catalog.get(5, 16400).unwrap(); + set.mark(parent.rfn, PendingReason::UnresolvedMultiXact); + set.mark(parent.rfn, PendingReason::ExternalToast); + + assert_eq!(set.len(), 1); + assert!(set.holds(5, 16400)); + assert_eq!(set.count_for(PendingReason::UnresolvedMultiXact), 1); + assert_eq!(set.count_for(PendingReason::ExternalToast), 0); + } + + /// Nothing pending means no source connection at all + #[tokio::test] + async fn repair_without_pending_relations_touches_nothing() { + let (tx, _rx) = mpsc::channel(1); + let source = crate::config::SourceConn::default().to_pg_config(); + let stats = repair( + &PendingSet::empty(), + &CatalogMap::new(), + &HashSet::new(), + &source, + 0x1000, + &tx, + ) + .await + .unwrap(); + assert_eq!(stats, RepairStats::default()); + } +} diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs new file mode 100644 index 00000000..6a6a7c39 --- /dev/null +++ b/src/backfill/wal_replay.rs @@ -0,0 +1,591 @@ +//! Decode bounded WAL ranges through shared transaction pipeline +//! +//! Used by greenfield window replay and object-store gap replay. Emit commits +//! above page-walk coverage and through each target's upper bound + +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use tokio::sync::{Mutex, mpsc}; +use walrus::pg::wal::segment::SegmentName; +use walrus::pg::walparser::{Oid, RmId}; + +use crate::budget::MemoryBudget; +use crate::catalog::desc_log::DescriptorLog; +use crate::config::ResolvedConfig; +use crate::decode::heap_decoder::CommittedTuple; +use crate::decode::visibility::PgXactPatch; +use crate::decode::wal_xact::{ + XLOG_XACT_ABORT, XLOG_XACT_ABORT_PREPARED, XLOG_XACT_ASSIGNMENT, XLOG_XACT_COMMIT, + XLOG_XACT_COMMIT_PREPARED, XLOG_XACT_OPMASK, parse_xact_assignment, parse_xact_payload, +}; +use crate::emit::ch_emitter::EmitterStats; +use crate::emit::pipeline::ack::AckHandle; +use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow}; +use crate::emit::route::{RouteSnapshot, freeze_routes}; +use crate::filter::manifest::Manifest; +use crate::mapping::MappingSnapshot; +use crate::record::{Record, RecordSink, SegmentSink, SinkError, WAL_SEG_SIZE}; +use crate::schema::{FIRST_NORMAL_OBJECT_ID, RelDescriptor, RelName}; +use crate::source::wal_stream::WalStream; +use crate::toast::{ChunkRefMap, ToastResolver}; +use crate::xact::xact_buffer::{ + BufferingDecoderSink, DrainEntry, DrainedBatch, SubxactTracker, WalkStep, XactBuffer, + detoast_heap, resolve_stash, +}; +use ahash::{HashMap, HashSet}; + +/// Per-filenode descriptor and exclusive replay ceiling +pub type ReplayTargets = HashMap<(Oid, Oid), (Arc, u64)>; + +/// Replay inputs shared across records +pub struct WalReplayInputs { + pub log: Arc, + pub buffer: Arc>, + pub resolver: ToastResolver, + /// Rfns whose heap records reach the decoder: targets plus their toast rels + pub filter_rfns: HashSet<(Oid, Oid)>, + pub targets: ReplayTargets, + /// Walk-coverage floor; commits at or below it drop + pub from_lsn: u64, + /// Treat unfiltered user filenodes as DDL when filter covers database + pub whole_db_filter: bool, + pub mapping: MappingSnapshot, + pub stats: Arc, + pub budget: Option, + pub row_policy: crate::emit::route::RowPolicy, + pub config: Option>, + pub batch_rows: usize, + pub batch_bytes: usize, + pub msg_tx: mpsc::Sender, + pub ack: AckHandle, + pub next_seq: u64, + /// Commit/abort overlay for walked-tuple visibility + pub patch: Option>>, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ReplayStats { + /// One past the last seq the leg registered + pub next_seq: u64, + pub rows_replayed: u64, + pub commits_past_through: u64, + /// Commits covered by walked pages + pub commits_below_from: u64, + /// Unknown user filenodes when filter covers database + pub unknown_rfns: u64, +} + +/// Serial replay drain over prefiltered records +pub struct WalReplaySink { + decoder: BufferingDecoderSink, + buffer: Arc>, + log: Arc, + /// Empty because replay applies committed history + pending: crate::catalog::pending::PendingCatalog, + subxact_tracker: SubxactTracker, + resolver: ToastResolver, + filter_rfns: HashSet<(Oid, Oid)>, + targets: ReplayTargets, + from_lsn: u64, + /// See [`WalReplayInputs::whole_db_filter`] + whole_db_filter: bool, + /// Routes frozen at replay start from the mapping + config snapshots + routes: HashMap>, + stats: Arc, + budget: Option, + /// Drain-slice budget, same knobs as the pipeline reorder + batch_rows: usize, + batch_bytes: usize, + msg_tx: mpsc::Sender, + ack: AckHandle, + patch: Option>>, + /// Current `(sequence, routed rows)`, registered on first row + open: Option<(u64, u64)>, + replay: ReplayStats, +} + +impl WalReplaySink { + pub fn new(inputs: WalReplayInputs) -> Self { + Self { + decoder: BufferingDecoderSink::new(inputs.log.clone(), inputs.buffer.clone()), + buffer: inputs.buffer, + log: inputs.log, + pending: Default::default(), + subxact_tracker: SubxactTracker::new(), + resolver: inputs.resolver, + filter_rfns: inputs.filter_rfns, + targets: inputs.targets, + from_lsn: inputs.from_lsn, + whole_db_filter: inputs.whole_db_filter, + routes: freeze_routes( + &inputs.mapping, + inputs.config.as_deref(), + &inputs.row_policy, + ), + stats: inputs.stats, + budget: inputs.budget, + batch_rows: inputs.batch_rows, + batch_bytes: inputs.batch_bytes, + msg_tx: inputs.msg_tx, + ack: inputs.ack, + patch: inputs.patch, + open: None, + replay: ReplayStats { + next_seq: inputs.next_seq, + ..Default::default() + }, + } + } + + /// Transactions needing replay below `lsn` to rebuild buffered prefix + pub async fn xacts_opened_below(&self, lsn: u64) -> Vec<(u32, u64)> { + self.buffer + .lock() + .await + .inflight_snapshot() + .into_iter() + .filter(|e| e.first_lsn < lsn) + .map(|e| (e.xid, e.first_lsn)) + .collect() + } + + pub fn stats(&self) -> ReplayStats { + self.replay + } + + async fn on_commit( + &mut self, + xid: u32, + info: u8, + record: &Record<'_>, + ) -> std::result::Result<(), SinkError> { + // Require subxact list to preserve buffered rows + let payload = parse_xact_payload(info, &record.parsed.main_data, record.page_magic) + .map_err(|e| SinkError::Other(format!("wal_replay: commit payload: {e}")))?; + // Prepared xid owns buffered work and visibility verdict + let xid = payload.twophase_xid.unwrap_or(xid); + if let Some(patch) = &self.patch { + patch + .lock() + .expect("wal_replay patch lock") + .commit(xid, &payload.subxacts); + } + // Resolve filenodes invisible at record time before drain + resolve_stash( + &self.buffer, + &self.log, + &self.pending, + xid, + &payload.subxacts, + record.next_lsn, + self.resolver.stats_handle(), + ) + .await + .map_err(SinkError::from)?; + let mut drain = self + .buffer + .lock() + .await + .drain_committed( + xid, + payload.xact_time, + record.source_lsn, + &payload.subxacts, + self.resolver.stores_chunks(), + ) + .await + .map_err(SinkError::from)?; + while let Some(batch) = drain + .next_batch(self.batch_rows, self.batch_bytes, self.budget.as_ref()) + .await + .map_err(SinkError::from)? + { + self.apply_batch(batch, drain.commit_ts, drain.commit_lsn) + .await?; + } + drain.finish().await.map_err(SinkError::from)?; + if let Some((seq, rows)) = self.open.take() { + self.ack.placed(seq, rows); + } + self.subxact_tracker.forget_tree(xid); + Ok(()) + } + + async fn apply_batch( + &mut self, + batch: DrainedBatch, + commit_ts: i64, + commit_lsn: u64, + ) -> std::result::Result<(), SinkError> { + let walk = batch.into_walk(); + let ref_maps: Vec<&ChunkRefMap> = walk.chunks.iter().map(|g| g.map()).collect(); + // One spool per transaction + let spool = walk.chunks.iter().find_map(|g| g.spool()); + let mut rows_cursor = 0usize; + for step in walk.steps { + match step { + WalkStep::Rows { upto } => { + if upto > rows_cursor { + self.resolver + .put_row_refs(walk.new_rows.spool(), &walk.new_rows[rows_cursor..upto]) + .await + .map_err(|e| SinkError::Other(format!("toast store put: {e}")))?; + rows_cursor = upto; + } + } + // Live stream owns DDL/config apply + WalkStep::Event(DrainEntry::Catalog(_)) + | WalkStep::Event(DrainEntry::Config(_)) => {} + WalkStep::Event(DrainEntry::ToastBarrier { + toast_relid, + marker_lsn, + }) => { + self.resolver + .rewrite_barrier(toast_relid, marker_lsn, commit_lsn) + .await + .map_err(|e| SinkError::Other(format!("toast rewrite barrier: {e}")))?; + } + WalkStep::Truncate(_) => { + // xl_heap_truncate carries no block ref, never passes the + // rfn filter + debug_assert!(false, "TRUNCATE heap in gap replay"); + } + WalkStep::Heap(mut heap) => { + let rfn = heap.decoded.rfn; + // Decode TOAST chunks, route through parent row + let Some((rel, through)) = self.targets.get(&(rfn.db_node, rfn.rel_node)) + else { + continue; + }; + if commit_lsn <= self.from_lsn { + // Walked pages cover commits through from_lsn + self.replay.commits_below_from += 1; + continue; + } + if commit_lsn > *through { + self.replay.commits_past_through += 1; + continue; + } + let Some(route) = self.routes.get(&rel.rel_name).cloned() else { + self.stats + .unsupported_relations + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + }; + // Skip deletes for append-only destinations + if route.drops_deletes() + && matches!(heap.decoded.op, crate::decode::heap_decoder::HeapOp::Delete) + { + self.stats + .deletes_discarded + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + } + let rel = rel.clone(); + let value_permit = detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) + .await + .map_err(SinkError::from)?; + let seq = if let Some((seq, rows)) = &mut self.open { + *rows += 1; + *seq + } else { + let seq = self.replay.next_seq; + self.replay.next_seq += 1; + self.ack.register(seq, commit_lsn); + self.open = Some((seq, 1)); + seq + }; + self.msg_tx + .send(BatcherMsg::Row(RoutedRow { + seq, + rel, + route, + committed: CommittedTuple { + decoded: heap.decoded, + commit_ts, + commit_lsn, + }, + value_permit: value_permit.map(Arc::new), + })) + .await + .map_err(|_| SinkError::Other("wal_replay: tail closed".into()))?; + self.replay.rows_replayed += 1; + } + } + } + Ok(()) + } +} + +impl RecordSink for WalReplaySink { + fn on_record<'a>( + &'a mut self, + record: &'a Record<'a>, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let rm = record.parsed.header.resource_manager_id; + if rm == RmId::Heap as u8 || rm == RmId::Heap2 as u8 { + if let Some(rel) = record.parsed.blocks.first().map(|b| b.header.location.rel) { + if self.filter_rfns.contains(&(rel.db_node, rel.rel_node)) { + self.decoder.on_record(record).await?; + } else if self.whole_db_filter + && rel.db_node == self.log.db_oid() + && rel.rel_node >= FIRST_NORMAL_OBJECT_ID + { + self.replay.unknown_rfns += 1; + } + } + } else if rm == RmId::Xact as u8 { + let info = record.parsed.header.info; + let xid = record.parsed.header.xact_id; + match info & XLOG_XACT_OPMASK { + XLOG_XACT_COMMIT | XLOG_XACT_COMMIT_PREPARED => { + self.on_commit(xid, info, record).await?; + } + XLOG_XACT_ABORT | XLOG_XACT_ABORT_PREPARED => { + let payload = + parse_xact_payload(info, &record.parsed.main_data, record.page_magic) + .map_err(|e| { + SinkError::Other(format!("wal_replay: abort payload: {e}")) + })?; + // ABORT PREPARED keys off the prepared xid too + let xid = payload.twophase_xid.unwrap_or(xid); + if let Some(patch) = &self.patch { + patch + .lock() + .expect("wal_replay patch lock") + .abort(xid, &payload.subxacts); + } + self.buffer + .lock() + .await + .abort(xid, record.source_lsn, &payload.subxacts) + .await + .map_err(SinkError::from)?; + self.subxact_tracker.forget_tree(xid); + } + XLOG_XACT_ASSIGNMENT => { + // Assignment only guides eviction policy + if let Some((xtop, subs)) = parse_xact_assignment(&record.parsed.main_data) + { + self.subxact_tracker.assign(xtop, &subs); + } + } + _ => { + // PREPARE / INVALIDATIONS unhandled; xact stays + // buffered until COMMIT_PREPARED + } + } + } + Ok(()) + }) + } +} + +/// Discard segment output while retaining record dispatch +pub struct DropSegments; + +impl SegmentSink for DropSegments { + fn on_segment<'a>( + &'a mut self, + _seg: SegmentName, + _bytes: &'a [u8], + _manifest: &'a Manifest, + ) -> Pin> + Send + 'a>> { + Box::pin(std::future::ready(Ok(()))) + } +} + +/// Drive fetched segments through `RecordSink` in LSN order +pub async fn pump_segments_through( + segments: &[(SegmentName, PathBuf)], + timeline: u32, + target_db_oid: Oid, + sink: &mut (dyn RecordSink + Send), +) -> Result<()> { + let Some((first, _)) = segments.first() else { + return Ok(()); + }; + let mut stream = WalStream::new(timeline, WAL_SEG_SIZE, first.start_lsn(WAL_SEG_SIZE)) + .map_err(|e| anyhow::anyhow!("wal_replay: WalStream: {e}"))?; + stream.filter_mut().set_target_db(target_db_oid); + let mut seg_sink = DropSegments; + for (seg, path) in segments { + let bytes = tokio::fs::read(path) + .await + .with_context(|| format!("read {}", path.display()))?; + stream + .push(seg.start_lsn(WAL_SEG_SIZE), &bytes, sink, &mut seg_sink) + .await + .map_err(|e| anyhow::anyhow!("wal_replay: {}: {e}", seg.format()))?; + } + stream + .close(None, sink) + .await + .map_err(|e| anyhow::anyhow!("wal_replay: close: {e}"))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::desc_log::DescLogIdentity; + use crate::decode::visibility::{PgXactAccum, PgXactView, XidStatus}; + use crate::decode::wal_xact::{ + XACT_XINFO_HAS_SUBXACTS, XACT_XINFO_HAS_TWOPHASE, XLOG_XACT_HAS_INFO, + }; + use crate::emit::pipeline::ack; + use crate::pos::{EmitterAck, Monotone}; + use crate::record::Record; + use std::path::Path; + use walrus::pg::walparser::{XLogRecord, XLogRecordHeader}; + + const DB: Oid = 5; + /// Backend that runs COMMIT PREPARED; not the xact that wrote the rows + const FINISHER_XID: u32 = 777; + const PREPARED_XID: u32 = 4242; + const PREPARED_SUBXID: u32 = 4243; + + fn xact_record(op: u8, xid: u32, subxacts: &[u32], twophase: Option) -> Record<'static> { + let mut md: Vec = 0i64.to_le_bytes().to_vec(); + let mut xinfo = 0u32; + if !subxacts.is_empty() { + xinfo |= XACT_XINFO_HAS_SUBXACTS; + } + if twophase.is_some() { + xinfo |= XACT_XINFO_HAS_TWOPHASE; + } + md.extend_from_slice(&xinfo.to_le_bytes()); + if !subxacts.is_empty() { + md.extend_from_slice(&(subxacts.len() as i32).to_le_bytes()); + for sub in subxacts { + md.extend_from_slice(&sub.to_le_bytes()); + } + } + if let Some(prepared) = twophase { + md.extend_from_slice(&prepared.to_le_bytes()); + } + Record { + parsed: XLogRecord { + header: XLogRecordHeader { + resource_manager_id: RmId::Xact as u8, + info: op | XLOG_XACT_HAS_INFO, + xact_id: xid, + ..Default::default() + }, + main_data: std::borrow::Cow::Owned(md), + ..Default::default() + }, + source_lsn: 0x5000, + next_lsn: 0x5100, + page_magic: 0xD116, + ..Default::default() + } + } + + /// Sink with no targets: the xact records under test carry no rows, so + /// only the patch and the buffer see them + async fn patch_sink(dir: &Path, patch: &Arc>) -> WalReplaySink { + let desc_dir = dir.join("desc_log"); + tokio::fs::create_dir_all(&desc_dir).await.unwrap(); + let log = DescriptorLog::open( + &desc_dir, + DescLogIdentity { + pg_major: 17, + system_id: "7300000000000000001".into(), + timeline: 1, + db_oid: DB, + wal_seg_size: WAL_SEG_SIZE as u32, + }, + ) + .await + .unwrap(); + let spill = dir.join("xact_spill"); + tokio::fs::create_dir_all(&spill).await.unwrap(); + let buffer = Arc::new(Mutex::new( + XactBuffer::new(crate::xact::xact_buffer::XactBufferConfig::new(spill)).unwrap(), + )); + let (msg_tx, _msg_rx) = mpsc::channel(8); + let (ack, _collector) = ack::spawn(Arc::new(Monotone::::new(0))); + WalReplaySink::new(WalReplayInputs { + log: Arc::new(log), + buffer, + resolver: ToastResolver::disabled(), + filter_rfns: HashSet::default(), + targets: ReplayTargets::default(), + from_lsn: 0, + whole_db_filter: false, + mapping: Arc::default(), + stats: Arc::new(EmitterStats::default()), + budget: None, + row_policy: Default::default(), + config: None, + batch_rows: 64, + batch_bytes: 1 << 20, + msg_tx, + ack, + next_seq: 0, + patch: Some(patch.clone()), + }) + } + + fn status(patch: &PgXactPatch, xid: u32) -> XidStatus { + let accum = PgXactAccum::new(); + PgXactView::new(&accum, patch).xid_status(xid) + } + + /// Patch prepared xid, not finishing backend xid + #[tokio::test] + async fn commit_prepared_patches_the_prepared_xid() { + let tmp = tempfile::tempdir().unwrap(); + let patch = Arc::new(std::sync::Mutex::new(PgXactPatch::new())); + let mut sink = patch_sink(tmp.path(), &patch).await; + sink.on_record(&xact_record( + XLOG_XACT_COMMIT_PREPARED, + FINISHER_XID, + &[PREPARED_SUBXID], + Some(PREPARED_XID), + )) + .await + .unwrap(); + let patch = patch.lock().unwrap(); + assert_eq!(status(&patch, PREPARED_XID), XidStatus::Committed); + assert_eq!(status(&patch, PREPARED_SUBXID), XidStatus::Committed); + assert_ne!(status(&patch, FINISHER_XID), XidStatus::Committed); + } + + #[tokio::test] + async fn abort_prepared_patches_the_prepared_xid() { + let tmp = tempfile::tempdir().unwrap(); + let patch = Arc::new(std::sync::Mutex::new(PgXactPatch::new())); + let mut sink = patch_sink(tmp.path(), &patch).await; + sink.on_record(&xact_record( + XLOG_XACT_ABORT_PREPARED, + FINISHER_XID, + &[], + Some(PREPARED_XID), + )) + .await + .unwrap(); + let patch = patch.lock().unwrap(); + assert_eq!(status(&patch, PREPARED_XID), XidStatus::Aborted); + assert_ne!(status(&patch, FINISHER_XID), XidStatus::Aborted); + } + + /// Reject payloads missing subxact list + #[tokio::test] + async fn malformed_xact_payload_stops_the_leg() { + let tmp = tempfile::tempdir().unwrap(); + let patch = Arc::new(std::sync::Mutex::new(PgXactPatch::new())); + let mut sink = patch_sink(tmp.path(), &patch).await; + let mut rec = xact_record(XLOG_XACT_COMMIT, 900, &[901], None); + rec.parsed.main_data = std::borrow::Cow::Owned(vec![0u8; 10]); + assert!(sink.on_record(&rec).await.is_err()); + let mut rec = xact_record(XLOG_XACT_ABORT, 900, &[901], None); + rec.parsed.main_data = std::borrow::Cow::Owned(vec![0u8; 10]); + assert!(sink.on_record(&rec).await.is_err()); + } +} diff --git a/src/bin/stream.rs b/src/bin/stream.rs index e59fd6f8..be291e85 100644 --- a/src/bin/stream.rs +++ b/src/bin/stream.rs @@ -48,6 +48,10 @@ use walrus::pg::backup::{BACKUP_NAME_PREFIX, format_pg_lsn}; use walrus::pg::replication::base_backup::BaseBackupOpts; use walrus::pg::replication::conn::PgConfig; use walrus::pg::replication::tls::SslMode; +use walshadow::backfill::visibility_gate::{ + GateStats, PendingGate, resolve_greenfield, stream_phase, +}; +use walshadow::backfill::visibility_repair::PendingSet; use walshadow::backfill_bootstrap::{ BootstrapConfig, BootstrapOutcome, drain_backfill, seed_in_snapshot, spawn_greenfield_bootstrap, }; @@ -66,7 +70,8 @@ use walshadow::manifest; use walshadow::mapping::{DropTableStrategy, MappingHandle}; use walshadow::metrics::{MetricsRegistry, MetricsSnapshot, RateEstimator}; use walshadow::pg::{quote_ident, socket_conninfo}; -use walshadow::pipeline::{Fatal, PipelineConfig, TailKind, bootstrap, tail}; +use walshadow::pipeline::tail::OwnedTail; +use walshadow::pipeline::{Fatal, PipelineConfig, TailKind, bootstrap}; use walshadow::pos::{ Drain, EmitterAck, FilterDispatched, FilterDurable, Floor, Gate, Monotone, Pos, ShadowFlush, ShadowReplay, SourceReceived, @@ -74,7 +79,9 @@ use walshadow::pos::{ use walshadow::queueing_record_sink::{ DEFAULT_QUEUEING_BATCH_SIZE, DEFAULT_QUEUEING_RECORD_SINK_CAPACITY, QueueingRecordSink, }; -use walshadow::record::{MetricsRecordSink, Record, RecordSink, SinkError, WAL_SEG_SIZE}; +use walshadow::record::{ + MetricsRecordSink, Record, RecordSink, SinkError, WAL_SEG_SIZE, segments_covering, +}; use walshadow::retention::{ DEFAULT_RETENTION_BYTES, DEFAULT_TRIM_INTERVAL, max_segment_end, trim_below_lsn, }; @@ -90,6 +97,7 @@ use walshadow::transition::{ CrossingState, CrossingWedge, ForkGuards, Switchover, TimelineStats, TransitionError, load_boot_history, seed_shadow_branches, }; +use walshadow::visibility::PgXactPatch; use walshadow::wal_stream::WalStream; use walshadow::xact_buffer::{BufferingDecoderSink, SubxactTracker, XactBuffer, XactBufferConfig}; @@ -100,6 +108,13 @@ struct BootstrapPlan { parallelism: Option, } +impl BootstrapPlan { + /// Stream window live for Direct mode, replay hydrated WAL otherwise + fn live_window_leg(&self, args: &Args) -> bool { + self.mode == BootstrapMode::Direct && !args.bootstrap_wal_from_archive + } +} + /// `cli_over_toml` plus a ≥1 clamp for pool/batch sizes. fn positive_usize(name: &str, cli: Option, toml: usize) -> usize { match cli_over_toml(cli, Some(toml)).unwrap_or(toml) { @@ -538,6 +553,11 @@ struct Args { /// cost matters more than bootstrap latency. #[arg(long, default_value_t = true)] bootstrap_fast_checkpoint: bool, + /// BASE_BACKUP `MAX_RATE` in kB/s for `direct` mode (PG accepts + /// 32..1048576). Caps the backup's read bandwidth on the source; the + /// window WAL leg keeps the destination converging while it streams + #[arg(long, value_parser = clap::value_parser!(i32).range(32..=1_048_576))] + bootstrap_max_rate_kib: Option, /// Fetch the bootstrap WAL window from the `[backup]` bucket instead of /// inside `base.tar` (`direct` mode only). Source then needn't retain or /// re-ship `[start_lsn, end_lsn]`, which is what fills its disk at high @@ -657,12 +677,16 @@ fn build_otlp_provider( fn init_tracing( otlp_endpoint: Option<&str>, ) -> Option { + use std::io::IsTerminal; + use opentelemetry::trace::TracerProvider as _; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; let fmt_layer = tracing_subscriber::fmt::layer() .with_target(true) + // Redirected stderr is read by tests and log collectors, not a pager + .with_ansi(std::io::stderr().is_terminal()) .with_writer(std::io::stderr); // Best-effort: a bad endpoint logs and degrades to no-traces rather @@ -947,29 +971,34 @@ async fn run_session( .with_context(|| format!("ensure physical replication slot {slot}"))?; tracing::info!(target: "walshadow", slot, "physical replication slot ready"); } - let bootstrap_end_lsn: Option = if matches!(shadow_start, ShadowStart::Bootstrap(_)) { - if !args.skip_preflight { - let source_sql = feed - .sql_client() - .await - .context("source sidecar sql for bootstrap pre-flight")?; - walshadow::preflight::bootstrap(walshadow::preflight::BootstrapInputs { - source_sql, - wal_from_archive: args.bootstrap_wal_from_archive, - }) - .await - .context("bootstrap pre-flight probe")? - .into_result() - .context("pre-flight rejected bootstrap")?; - } - Some( - run_bootstrap(&cfg, &mut feed, args, &bootstrap_plan, ch_config.clone()) + let bootstrap_handoff: Option = + if matches!(shadow_start, ShadowStart::Bootstrap(_)) { + if !args.skip_preflight { + let source_sql = feed + .sql_client() + .await + .context("source sidecar sql for bootstrap pre-flight")?; + walshadow::preflight::bootstrap(walshadow::preflight::BootstrapInputs { + source_sql, + wal_from_archive: args.bootstrap_wal_from_archive, + window_leg: bootstrap_plan.live_window_leg(args), + }) .await - .context("bootstrap")?, - ) - } else { - None - }; + .context("bootstrap pre-flight probe")? + .into_result() + .context("pre-flight rejected bootstrap")?; + } + Some( + run_bootstrap(&cfg, &mut feed, args, &bootstrap_plan, ch_config.clone()) + .await + .context("bootstrap")?, + ) + } else { + None + }; + let bootstrap_end_lsn: Option = bootstrap_handoff.as_ref().map(|h| h.end_lsn); + let bootstrap_resume_lsn: Option = + bootstrap_handoff.as_ref().map(BootstrapHandoff::resume_lsn); // Regenerate config because shadow's port, socket, and GUC floor may change // Keep shadow alive until pipeline teardown finishes let shadow_lifecycle: Option = match &shadow_start { @@ -1031,10 +1060,7 @@ async fn run_session( ); } }; - // Resume precedence: `--start-lsn` > bootstrap end > manifest emitter-ack - // > greenfield (source write head). `--ignore-cursor` forces greenfield - // (recovery drills). Bootstrap `end_lsn` outranks the manifest: shadow - // catalog state is at `end_lsn`, so consuming WAL before it double-counts. + // Precedence: explicit > bootstrap > manifest > greenfield head let manifest_at_boot = if args.ignore_cursor { None } else { @@ -1042,7 +1068,7 @@ async fn run_session( }; let raw_start = manifest::resolve_resume_lsn( start_lsn_override, - bootstrap_end_lsn.map(Pos::new), + bootstrap_resume_lsn.map(Pos::new), manifest_at_boot.as_ref().map(|m| m.lsn.emitter_ack), ident.xlogpos, ); @@ -1342,14 +1368,11 @@ async fn run_session( "spill dir ready", ); - // Persist the bootstrap end LSN as the initial resume manifest so a restart - // in the window before the first manifest write resumes from `end_lsn` (like - // a standby reading its backup's redo point) instead of falling to greenfield - // head and skipping `(end_lsn, head]`. Only after a fresh bootstrap. - if let Some(end_lsn) = bootstrap_end_lsn { + // Persist handoff before streaming can advance manifest + if let (Some(end_lsn), Some(resume)) = (bootstrap_end_lsn, bootstrap_resume_lsn) { let initial = manifest::Manifest { version: manifest::MANIFEST_VERSION, - floor: manifest::resolved_floor(end_lsn, end_lsn), + floor: manifest::resolved_floor(resume, end_lsn), source: live_identity.clone(), wal: manifest::WalBranch { stream_timeline: start_timeline, @@ -1358,8 +1381,8 @@ async fn run_session( source_received: end_lsn.into(), filter_durable: end_lsn.into(), shadow_replay: end_lsn.into(), - drain: end_lsn.into(), - emitter_ack: end_lsn.into(), + drain: resume.into(), + emitter_ack: resume.into(), shadow_flush: end_lsn.into(), }, }; @@ -4185,7 +4208,7 @@ async fn run_bootstrap( args: &Args, plan: &BootstrapPlan, ch_config: Option, -) -> Result { +) -> Result { let shadow_data_dir = args .bootstrap_shadow_data_dir .clone() @@ -4244,7 +4267,7 @@ async fn run_bootstrap( ), fast_checkpoint: args.bootstrap_fast_checkpoint, no_verify_checksums: false, - max_rate_kib: None, + max_rate_kib: args.bootstrap_max_rate_kib, wal: hydrate.is_none(), }; (Box::new(DirectSource::new(src_cfg.clone(), opts)), hydrate) @@ -4296,34 +4319,90 @@ async fn run_bootstrap( }; let store_toast = resolver.stores_chunks(); - let ch_target = match ch_config { + let (ch_target, walk_skip) = match ch_config { Some(emitter_cfg) => { let (mapping, resolved) = bootstrap_build_mapping(&emitter_cfg, &drain_catalog, args) .await .context("bootstrap: build mapping")?; - Some((emitter_cfg, mapping, resolved)) + let routed = mapping.snapshot().await; + let skip_initial: HashSet<_> = drain_catalog + .descriptors() + .filter_map(|d| { + let rn = &d.rel_name; + let table_mode = emitter_cfg + .table_opt_ins + .get(rn) + .and_then(|r| r.initial_load.as_deref()) + .or_else(|| emitter_cfg.table_initial_loads.get(rn).map(String::as_str)); + let none = match table_mode { + Some(s) => s.parse::() == Ok(InitialLoadMode::None), + None => { + resolved + .namespaces + .get(rn.namespace.as_ref()) + .and_then(|n| n.initial_load) + == Some(InitialLoadMode::None) + } + }; + none.then(|| rn.clone()) + }) + .collect(); + let mapped: HashSet<_> = drain_catalog + .descriptors() + .filter(|d| routed.contains_key(&d.rel_name) && !skip_initial.contains(&d.rel_name)) + .map(|d| (d.rfn.db_node, d.rfn.rel_node)) + .collect(); + let pending = PendingSet::mapped(&drain_catalog, &routed).scoped_to(mapped.clone()); + let skip = drain_catalog + .descriptors() + .map(|d| (d.rfn.db_node, d.rfn.rel_node)) + .filter(|&(db, rel)| { + !drain_catalog.is_toast(db, rel) && !mapped.contains(&(db, rel)) + }) + .collect(); + ( + Some((emitter_cfg, mapping, resolved, pending, skip_initial)), + skip, + ) } - None => None, + None => (None, HashSet::default()), }; prepare_bootstrap_dir(&shadow_data_dir) .await .context("prepare shadow data dir for bootstrap")?; - let cfg = BootstrapConfig::new(shadow_data_dir.clone()); + // Sample window floor before BASE_BACKUP + let source_ident = feed + .identify_system() + .await + .context("bootstrap: sample source write head for the window leg")?; + let source_major = (feed.server_version_num() / 10000) as u32; + + let cfg = BootstrapConfig::new(shadow_data_dir.clone()).with_skip_filenodes(walk_skip); let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); - let (shipped, outcome) = if let Some((emitter_cfg, mapping, resolved)) = ch_target { + // Keep oracle alive through live or file window replay + let mut bootstrap_oracle: Option = None; + let mut window_cfg: Option = None; + // Overlay window transaction outcomes on backup pg_xact + let window_patch = Arc::new(std::sync::Mutex::new(PgXactPatch::new())); + // Metrics-only mode has no pending gate + let mut pending_gate: Option = None; + // Preserve live failure if file fallback also fails + let mut window_leg_error: Option = None; + let window_scratch = args.spill_dir.join("bootstrap_window"); + tokio::fs::remove_dir_all(&window_scratch).await.ok(); + + let (shipped, outcome, mut window) = if let Some(target) = ch_target { + let (emitter_cfg, mapping, resolved, mut pending, skip_initial) = target; // Route bootstrap rows through the shared insert tail. Bootstrap // is the easy case: every row op=Insert at _lsn = start_lsn, no // aborts / TRUNCATE / DDL. Keep operator's flush_timeout; tail // defaults 0 to its own partial-flush deadline. let addr = format!("{}:{}", emitter_cfg.host, emitter_cfg.port); let stats = bootstrap_stats.clone(); - // Throwaway watermark: durability proof is `wait_through(K)`, - // resume LSN is carried via the WAL pipeline's emitter_ack seed - // (see `run`), so uniform `commit_lsn = start_lsn` here is fine. - let emitter_ack = Arc::new(Monotone::::new(0)); + // Window leg shares the tail's fatal, so a CH outage stops both let fatal = Fatal::new(); let inserter_pool_size = emitter_cfg.inserter_pool_size; @@ -4339,12 +4418,12 @@ async fn run_bootstrap( "prefer" }, ); - let bootstrap_oracle = if walshadow::backfill::bootstrap_oracle::needs_oracle( + if walshadow::backfill::bootstrap_oracle::needs_oracle( &drain_catalog, &mapping.snapshot().await, &resolved.column_rules, ) { - Some( + bootstrap_oracle = Some( walshadow::backfill::bootstrap_oracle::BootstrapOracle::provision( args.spill_dir.join("bootstrap_oracle"), source_conninfo, @@ -4357,22 +4436,23 @@ async fn run_bootstrap( "bootstrap oracle: greenfield needs it to convert oracle columns; \ refusing to load empty columns", )?, - ) - } else { - None - }; + ); + } - let (msg_tx, ack, tail) = tail::spawn_with_config( + // Throwaway watermark: durability proof is `wait_through(K)`, resume + // LSN is carried via the WAL pipeline's emitter_ack seed (see `run`), + // so uniform `commit_lsn = start_lsn` here is fine. + let tail = OwnedTail::spawn( &emitter_cfg, inserter_pool_size, stats.clone(), - emitter_ack, fatal.clone(), None, bootstrap_oracle.as_ref().map(|o| o.oracle()), + "bootstrap", ) .await - .context("bootstrap: spawn insert tail")?; + .map_err(anyhow::Error::msg)?; tracing::info!( target: "walshadow::bootstrap", addr = %addr, @@ -4380,40 +4460,63 @@ async fn run_bootstrap( "bootstrap insert tail started", ); - // `initial_load = "none"` (table override, else namespace) opts a - // relation out of the greenfield snapshot: create it + stream CDC, but - // don't page-walk its existing rows. - let skip_initial: std::collections::HashSet<_> = drain_catalog - .descriptors() - .filter_map(|d| { - let rn = &d.rel_name; - let table_mode = emitter_cfg - .table_opt_ins - .get(rn) - .and_then(|r| r.initial_load.as_deref()) - .or_else(|| emitter_cfg.table_initial_loads.get(rn).map(String::as_str)); - let none = match table_mode { - Some(s) => s.parse::() == Ok(InitialLoadMode::None), - None => { - resolved - .namespaces - .get(rn.namespace.as_ref()) - .and_then(|n| n.initial_load) - == Some(InitialLoadMode::None) - } - }; - none.then(|| rn.clone()) - }) - .collect(); + // Run WAL window beside page walk when user relations exist + window_cfg = (!drain_catalog.is_empty()).then(|| { + walshadow::backfill::bootstrap_window::WindowLegConfig { + emitter: emitter_cfg.clone(), + mapping: mapping.clone(), + config: resolved.clone(), + stats: stats.clone(), + resolver: resolver.clone(), + oracle: bootstrap_oracle.as_ref().map(|o| o.oracle()), + fatal: fatal.clone(), + scratch_dir: window_scratch.clone(), + patch: window_patch.clone(), + catalog: drain_catalog.clone(), + pg_major: source_major, + system_id: source_ident.sysid.clone(), + timeline: source_ident.timeline, + } + }); + let live_cfg = window_cfg.clone().filter(|_| plan.live_window_leg(args)); + + // Gate page tuples, defer unknowns until transaction logs land + let gate_spool_path = args.spill_dir.join("bootstrap_gate_deferred.bin"); + tokio::fs::remove_file(&gate_spool_path).await.ok(); + let (gated_tx, gated_rx) = + tokio::sync::mpsc::channel::( + walshadow::backup_page_walk::BOOTSTRAP_TUPLE_CHANNEL_CAP, + ); + let gate = tokio::spawn({ + let catalog = drain_catalog.clone(); + let mut rx = rx; + let mut spool = walshadow::spool::DeferredSpool::new( + gate_spool_path, + walshadow::spool::DEFERRED_SPOOL_MEM_MAX, + ); + async move { + let mut gate_stats = GateStats::default(); + stream_phase( + &mut rx, + &gated_tx, + &catalog, + &mut pending, + &mut spool, + &mut gate_stats, + ) + .await + .map(|()| (gate_stats, spool, pending)) + } + }); let deferred_path = args.spill_dir.join("bootstrap_deferred.bin"); tokio::fs::remove_file(&deferred_path).await.ok(); let drain = tokio::spawn(bootstrap::drain( - rx, - drain_catalog, - mapping, - msg_tx.clone(), - ack.clone(), + gated_rx, + drain_catalog.clone(), + mapping.clone(), + tail.msg_tx.clone(), + tail.ack.clone(), stats.clone(), resolver.clone(), walshadow::spool::DeferredSpool::new( @@ -4424,21 +4527,68 @@ async fn run_bootstrap( // No source-PG overlay during greenfield bootstrap, but the same // snapshot the CREATEs above rendered from: per-relation system // column names have to match what CH now holds - Some(resolved), - skip_initial, + Some(resolved.clone()), + skip_initial.clone(), )); - let (drain_res, pump_res) = tokio::join!(drain, pump); + // Borrow feed until stop watch publishes end_lsn or zero + let (stop_tx, stop_rx) = tokio::sync::watch::channel(None); + // Keep sender alive while leg winds down + let stop_tx = &stop_tx; + let pump_then_stop = async move { + let res = pump.await; + let end = match &res { + Ok(Ok(o)) => o.end.end_lsn, + _ => 0, + }; + let _ = stop_tx.send(Some(end)); + res + }; + let leg_fut = async { + match live_cfg { + Some(cfg) => walshadow::backfill::bootstrap_window::stream_window( + cfg, + feed, + source_ident.xlogpos, + stop_rx, + ) + .await + .map(Some), + None => Ok(None), + } + }; + let (gate_res, drain_res, pump_res, leg_res) = + tokio::join!(gate, drain, pump_then_stop, leg_fut); + let (gate_stats, gate_spool, pending) = gate_res + .context("bootstrap gate join")? + .map_err(|e| anyhow::anyhow!("bootstrap gate: {e}"))?; let drain_outcome = drain_res .context("bootstrap drain join")? .map_err(|e| anyhow::anyhow!("bootstrap drain: {e}"))?; let outcome: BootstrapOutcome = pump_res .context("bootstrap pump join")? .context("bootstrap pump")?; + // Retry failed live read from landed WAL + let window = match leg_res { + Ok(w) => { + if w.is_some() { + window_cfg = None; + } + w + } + Err(e) => { + tracing::warn!( + target: "walshadow::bootstrap", + error = %format!("{e:#}"), + "live backup-window leg failed; replaying the window from the \ + WAL the backup landed", + ); + window_leg_error = Some(e); + None + } + }; let k = drain_outcome.next_seq; - tail.finish(msg_tx, ack, k, &fatal) - .await - .map_err(|m| anyhow::anyhow!("bootstrap: {m}"))?; + tail.finish(k).await.map_err(anyhow::Error::msg)?; tracing::info!( target: "walshadow::bootstrap", rows_routed = drain_outcome.rows_routed, @@ -4447,18 +4597,60 @@ async fn run_bootstrap( seqs = k, "bootstrap insert tail drained", ); - (drain_outcome.rows_routed, outcome) + pending_gate = Some(PendingGate { + deferred: gate_spool, + pending, + catalog: drain_catalog, + mapping, + config: resolved, + emitter: emitter_cfg, + stats, + resolver: resolver.clone(), + oracle: bootstrap_oracle.as_ref().map(|o| o.oracle()), + skip_initial, + source: src_cfg.clone(), + start_lsn: outcome.start.start_lsn, + spill_dir: args.spill_dir.clone(), + stream_stats: gate_stats, + }); + (drain_outcome.rows_routed, outcome, window) } else { - // Metrics-only: bootstrap rows counted, not shipped. + // Metrics-only skips destination convergence let mut observer = MetricsTupleObserver::default(); let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &mut observer), pump); let shipped = drain_res.context("bootstrap drain")?; let outcome: BootstrapOutcome = pump_res .context("bootstrap pump join")? .context("bootstrap pump")?; - (shipped, outcome) + (shipped, outcome, None) }; + // Replace live-leg COPY connection and recheck source identity + if window.is_some() || window_leg_error.is_some() { + *feed = SourceFeed::connect(src_cfg) + .await + .with_context(|| { + format!( + "bootstrap: reconnect source {}:{} after the window leg", + src_cfg.host, src_cfg.port + ) + })? + .with_status_interval(Duration::from_secs(args.status_interval)); + let now = feed + .identify_system() + .await + .context("bootstrap: IDENTIFY_SYSTEM after the window leg")?; + anyhow::ensure!( + now.sysid == source_ident.sysid && now.timeline == source_ident.timeline, + "source identity moved during bootstrap: system {} timeline {} when the backup \ + opened, system {} timeline {} now", + source_ident.sysid, + source_ident.timeline, + now.sysid, + now.timeline, + ); + } + tracing::info!( target: "walshadow::bootstrap", start_lsn = format_pg_lsn(outcome.start.start_lsn).to_string(), @@ -4467,6 +4659,7 @@ async fn run_bootstrap( kept_files = outcome.disk.kept_files, skipped_denylist = outcome.disk.skipped_denylist, files_walked = outcome.page_walk.files_walked, + files_skipped_by_caller = outcome.page_walk.files_skipped_by_caller, tuples_emitted = outcome.page_walk.tuples_emitted, drained = shipped, "bootstrap landed", @@ -4485,6 +4678,70 @@ async fn run_bootstrap( .context("bootstrap: hydrate shadow pg_wal from object store")?; } + // Replay hydrated or fallback window from shadow pg_wal + if let Some(mut cfg) = window_cfg { + cfg.timeline = outcome.start.timeline; + let pg_wal = shadow_data_dir.join("pg_wal"); + let replayed = async { + let segments = walshadow::backfill::bootstrap_window::segments_in_dir( + &pg_wal, + outcome.start.timeline, + outcome.start.start_lsn, + outcome.end.end_lsn, + ) + .await?; + walshadow::backfill::bootstrap_window::replay_segments( + cfg, + &segments, + outcome.start.start_lsn, + outcome.end.end_lsn, + ) + .await + } + .await; + window = Some(replayed.map_err(|e| match window_leg_error { + Some(live) => e.context(format!("after the live window leg failed: {live:#}")), + None => e.context("bootstrap: backup-window WAL leg"), + })?); + } + let open_floor = window.and_then(|w| w.open_floor); + if let Some(w) = window { + tracing::info!( + target: "walshadow::bootstrap", + from_lsn = format_pg_lsn(w.from_lsn).to_string(), + through_lsn = format_pg_lsn(w.through_lsn).to_string(), + rows = w.replay.rows_replayed, + commits_below_from = w.replay.commits_below_from, + unknown_rfns = w.replay.unknown_rfns, + open_floor = w.open_floor.map(|l| format_pg_lsn(l).to_string()), + "backup window shipped", + ); + } + tokio::fs::remove_dir_all(&window_scratch).await.ok(); + + // Resolve deferred tuples after window transaction overlay is complete + if let Some(pending) = pending_gate { + let patch = std::mem::take(&mut *window_patch.lock().expect("window patch lock")); + let gate = resolve_greenfield(pending, &shadow_data_dir, &patch) + .await + .context("bootstrap: visibility gate")?; + tracing::info!( + target: "walshadow::bootstrap", + emitted = gate.emitted, + gated = gate.gated, + deferred = gate.deferred, + multixact_emitted = gate.multixact_emitted, + chunks_gated = gate.chunks_gated, + pending_relations = gate.pending_relations, + pending_multixact = gate.pending_multixact, + pending_discarded = gate.pending_discarded, + repaired_rows = gate.repaired_rows, + p_hi = gate.p_hi, + patch_xacts = patch.len(), + "bootstrap visibility gate settled", + ); + } + // PG refuses to start on a data dir whose mode isn't 0700 or 0750. // BASE_BACKUP tar carries no entry for the root, so extraction leaves // it at the process umask (typically 0755); reassert 0700 before pg_ctl. @@ -4501,7 +4758,25 @@ async fn run_bootstrap( .await .context("clear completed bootstrap marker")?; - Ok(outcome.end.end_lsn) + Ok(BootstrapHandoff { + end_lsn: outcome.end.end_lsn, + open_floor, + }) +} + +/// Bootstrap-to-pump handoff +struct BootstrapHandoff { + /// Backup end and shadow state boundary + end_lsn: u64, + /// Earliest record among transactions open at window seal + open_floor: Option, +} + +impl BootstrapHandoff { + /// Source or archive must retain crossing transaction records + fn resume_lsn(&self) -> u64 { + self.open_floor.unwrap_or(self.end_lsn).min(self.end_lsn) + } } /// Routing map for the bootstrap drain: explicit `[table.*]` seeded up front, @@ -4929,25 +5204,16 @@ async fn fetch_wal_into_pg_wal( end_lsn: u64, timeline: u32, ) -> Result<()> { - use walrus::pg::wal::segment::SegmentName; - - let seg_size = WAL_SEG_SIZE; let pg_wal_dir = shadow_data_dir.join("pg_wal"); tokio::fs::create_dir_all(&pg_wal_dir) .await .with_context(|| format!("create {}", pg_wal_dir.display()))?; - let mut cur = SegmentName { - timeline, - log_id: (start_lsn >> 32) as u32, - seg_no: ((start_lsn & 0xFFFF_FFFF) / seg_size) as u32, - }; - let mut fetched: u32 = 0; - loop { - let name = cur.format(); + let segments = segments_covering(timeline, start_lsn..end_lsn.saturating_add(1)); + for seg in &segments { + let name = seg.format(); let dst = pg_wal_dir.join(&name); - // Off: loop enumerates every segment in [start,end] explicitly, so - // read-ahead would only duplicate the next iteration's fetch & risk - // downloading past end_lsn + // Off: the range is enumerated explicitly, so read-ahead would only + // duplicate the next fetch & risk downloading past end_lsn walrus::pg::wal::fetch::handle( settings, storage.clone(), @@ -4957,16 +5223,10 @@ async fn fetch_wal_into_pg_wal( ) .await .with_context(|| format!("fetch WAL {name} -> {}", dst.display()))?; - fetched += 1; - let seg_end = cur.start_lsn(seg_size).saturating_add(seg_size); - if end_lsn < seg_end { - break; - } - cur = cur.next(seg_size); } tracing::info!( target: "walshadow::bootstrap", - fetched, + fetched = segments.len(), start_lsn = format_pg_lsn(start_lsn).to_string(), end_lsn = format_pg_lsn(end_lsn).to_string(), timeline, @@ -5301,4 +5561,27 @@ mod tests { assert!(ci.contains("host=127.0.0.1"), "{ci}"); assert!(ci.contains("port=5441"), "{ci}"); } + + #[test] + fn bootstrap_handoff_preserves_required_history() { + let crossing = BootstrapHandoff { + end_lsn: 0x3000, + open_floor: Some(0x1000), + }; + assert_eq!(crossing.resume_lsn(), 0x1000); + + let clean = BootstrapHandoff { + end_lsn: 0x3000, + open_floor: None, + }; + assert_eq!(clean.resume_lsn(), 0x3000); + assert_eq!( + BootstrapHandoff { + end_lsn: 0x3000, + open_floor: Some(0x4000) + } + .resume_lsn(), + 0x3000 + ); + } } diff --git a/src/catalog/shadow.rs b/src/catalog/shadow.rs index 4d812f80..2c82c06c 100644 --- a/src/catalog/shadow.rs +++ b/src/catalog/shadow.rs @@ -754,7 +754,7 @@ fn log_tail(path: &Path) -> String { }; let len = f.metadata().map(|m| m.len()).unwrap_or(0); let _ = f.seek(SeekFrom::Start(len.saturating_sub(TAIL))); - let mut buf = Vec::new(); + let mut buf = Vec::with_capacity(len.min(TAIL) as usize); let _ = f.read_to_end(&mut buf); String::from_utf8_lossy(&buf).into_owned() } diff --git a/src/catalog/shadow_catalog.rs b/src/catalog/shadow_catalog.rs index f5b2dc30..7eafac8a 100644 --- a/src/catalog/shadow_catalog.rs +++ b/src/catalog/shadow_catalog.rs @@ -812,9 +812,8 @@ fn descriptor_from_rows( class.relnamespace, class.oid )) })?; - let replident = replident_from_parts( + let replident = ReplIdent::from_parts( class.relreplident, - class.oid, indexes .iter() .find(|i| i.indisprimary) @@ -823,7 +822,8 @@ fn descriptor_from_rows( .iter() .find(|i| i.indisreplident) .map(|i| (i.indexrelid, i.indkey.clone())), - )?; + ) + .map_err(|e| CatalogError::Parse(format!("{e} for relation {}", class.oid)))?; let mut ordered: Vec<&AttributeRow> = attrs.iter().collect(); ordered.sort_unstable_by_key(|a| a.attnum); @@ -878,33 +878,6 @@ fn descriptor_from_rows( }) } -fn replident_from_parts( - c: char, - rel_oid: Oid, - pk_attnums: Option>, - using_index: Option<(Oid, Vec)>, -) -> Result { - match c { - 'd' => Ok(ReplIdent::Default { pk_attnums }), - 'n' => Ok(ReplIdent::Nothing), - 'f' => Ok(ReplIdent::Full { pk_attnums }), - 'i' => { - let (index_oid, key_attnums) = using_index.ok_or_else(|| { - CatalogError::Parse(format!( - "relreplident='i' but no pg_index row with indisreplident=true for relation {rel_oid}", - )) - })?; - Ok(ReplIdent::UsingIndex { - index_oid, - key_attnums, - }) - } - other => Err(CatalogError::Parse(format!( - "unknown relreplident {other:?} (expected one of d/n/f/i)", - ))), - } -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/src/decode/codecs.rs b/src/decode/codecs.rs index 153845c1..ded4cf85 100644 --- a/src/decode/codecs.rs +++ b/src/decode/codecs.rs @@ -59,7 +59,7 @@ pub fn decode_text_array(body: &[u8]) -> Option> { let nitems = usize::try_from(word(12)?).ok()?; // ARR_DATA_PTR: MAXALIGN(sizeof(ArrayType) + 2 * 4 * ndim) less stripped header let mut cursor = 20usize; - let mut values = Vec::new(); + let mut values = Vec::with_capacity(nitems); for _ in 0..nitems { cursor = cursor.next_multiple_of(4); // Elements keep their own varlena header, nominal-aligned (`array_out`) @@ -189,7 +189,7 @@ pub fn decode_numeric(body: &[u8]) -> Result { (sign, weight, dscale, 4usize) }; - let mut digits = Vec::new(); + let mut digits = Vec::with_capacity((body.len() - digits_off) / 2); let mut cur = digits_off; while cur + 2 <= body.len() { digits.push(i16::from_le_bytes([body[cur], body[cur + 1]])); diff --git a/src/decode/visibility.rs b/src/decode/visibility.rs index 39e35277..d16adb26 100644 --- a/src/decode/visibility.rs +++ b/src/decode/visibility.rs @@ -29,7 +29,10 @@ //! risks resurrecting a pre-coverage dead version, skipping risks dropping //! a live row, so the pass aborts. +use std::path::{Path, PathBuf}; + use ahash::{HashMap, HashSet}; +use anyhow::{Context, Result}; // t_infomask bits, PG src/include/access/htup_details.h pub const HEAP_XMAX_KEYSHR_LOCK: u16 = 0x0010; @@ -448,7 +451,7 @@ pub fn tuple_visibility( } /// Parse a `pg_xact/` cluster-relative path into its segment number. -pub fn pg_xact_segno_from_path(path: &std::path::Path) -> Option { +pub fn pg_xact_segno_from_path(path: &Path) -> Option { let mut comps = path.components(); let dir = comps.next()?; if dir.as_os_str() != "pg_xact" { @@ -468,7 +471,7 @@ pub enum MultiXactSegment { } /// Parse a `pg_multixact/{offsets,members}/` cluster-relative path. -pub fn pg_multixact_segno_from_path(path: &std::path::Path) -> Option { +pub fn pg_multixact_segno_from_path(path: &Path) -> Option { let mut comps = path.components(); if comps.next()?.as_os_str() != "pg_multixact" { return None; @@ -486,10 +489,68 @@ pub fn pg_multixact_segno_from_path(path: &std::path::Path) -> Option Result { + let mut accum = PgXactAccum::new(); + for (rel, bytes) in read_slru_dir(data_dir, Path::new("pg_xact")).await? { + if let Some(segno) = pg_xact_segno_from_path(&rel) { + accum.insert_segment(segno, bytes); + } + } + Ok(accum) +} + +/// Read landed `pg_multixact` offsets and members segments +pub async fn read_pg_multixact(data_dir: &Path) -> Result { + let mut accum = PgMultiXactAccum::new(); + for sub in ["offsets", "members"] { + let dir = Path::new("pg_multixact").join(sub); + for (rel, bytes) in read_slru_dir(data_dir, &dir).await? { + match pg_multixact_segno_from_path(&rel) { + Some(MultiXactSegment::Offsets(s)) => accum.insert_offsets_segment(s, bytes), + Some(MultiXactSegment::Members(s)) => accum.insert_members_segment(s, bytes), + None => {} + } + } + } + Ok(accum) +} + +/// Read files under a cluster-relative SLRU directory, keyed by that path. +/// A directory the cluster never made reads as empty +async fn read_slru_dir(data_dir: &Path, rel_dir: &Path) -> Result)>> { + let dir = data_dir.join(rel_dir); + let mut entries = match tokio::fs::read_dir(&dir).await { + Ok(e) => e, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e).with_context(|| format!("slru: read {}", dir.display())), + }; + let mut out = Vec::new(); + while let Some(entry) = entries + .next_entry() + .await + .with_context(|| format!("slru: scan {}", dir.display()))? + { + let path = entry.path(); + if !entry + .file_type() + .await + .with_context(|| format!("slru: stat {}", path.display()))? + .is_file() + { + continue; + } + let bytes = tokio::fs::read(&path) + .await + .with_context(|| format!("slru: read {}", path.display()))?; + out.push((rel_dir.join(entry.file_name()), bytes)); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; - use std::path::Path; fn accum_with(segno: u32, statuses: &[(u32, u8)]) -> PgXactAccum { let mut bytes = vec![0u8; 8192]; @@ -794,4 +855,40 @@ mod tests { None ); } + + /// pg_xact segment bytes with `xid` marked committed + fn pg_xact_segment(xid: u32) -> Vec { + let mut bytes = vec![0u8; 8192]; + bytes[(xid / 4) as usize] |= 0x01 << ((xid % 4) * 2); + bytes + } + + #[tokio::test] + async fn slru_reads_come_off_the_landed_data_dir() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + tokio::fs::create_dir_all(dir.join("pg_xact")) + .await + .unwrap(); + tokio::fs::write(dir.join("pg_xact").join("0000"), pg_xact_segment(700)) + .await + .unwrap(); + // Segment 1 proves the hex filename is the segment number, not an index + tokio::fs::write(dir.join("pg_xact").join("0001"), pg_xact_segment(3)) + .await + .unwrap(); + + let accum = read_pg_xact(dir).await.unwrap(); + // No pg_multixact/ at all: a cluster that never made one + let multi = read_pg_multixact(dir).await.unwrap(); + let patch = PgXactPatch::new(); + let view = PgXactView::new(&accum, &patch).with_multixact(&multi); + + assert_eq!(view.xid_status(700), XidStatus::Committed); + assert_eq!( + view.xid_status(PG_XACT_XIDS_PER_SEGMENT + 3), + XidStatus::Committed, + ); + assert_eq!(view.xid_status(701), XidStatus::InProgress); + } } diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 786c1485..5789999b 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -16,7 +16,7 @@ use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::ack::AckHandle; use crate::emit::pipeline::batcher::{BatcherMsg, RoutedRow}; -use crate::emit::route::{RouteSnapshot, RowPolicy}; +use crate::emit::route::{RouteSnapshot, RowPolicy, freeze_routes}; use crate::mapping::{MappingHandle, TableMapping}; use crate::ops::oracle::render_ext_columns; use crate::schema::{RelDescriptor, RelName}; @@ -24,8 +24,7 @@ use crate::toast::{ CHUNK_PUT_BATCH, CHUNK_PUT_BYTES, FetchedValue, ToastResolver, ToastRow, check_value_caps, detoasted_value, finish_value, pointer_extsize, }; -use ahash::HashMap; -use std::collections::HashSet; +use ahash::HashSet; /// Completion frontier for `FlushAll` and resume advance #[derive(Debug, Clone, Copy, Default)] @@ -53,21 +52,11 @@ pub async fn drain( skip_initial: HashSet, ) -> Result { // Routes frozen once per pass from the caller's config snapshot - let routes: HashMap<_, _> = mapping_handle - .snapshot() - .await - .iter() - .map(|(name, mapping)| { - let rules = config - .as_ref() - .map_or_else(Arc::default, |rc| rc.column_rules.clone()); - let policy = row_policy.for_rel(config.as_deref(), name); - ( - name.clone(), - RouteSnapshot::freeze(Arc::new(mapping.clone()), rules, policy), - ) - }) - .collect(); + let routes = freeze_routes( + &mapping_handle.snapshot().await, + config.as_deref(), + &row_policy, + ); let mut next_seq = 0u64; let mut rows_routed = 0u64; let mut open: Option<(walrus::pg::walparser::RelFileNode, u64, u64)> = None; @@ -364,7 +353,7 @@ mod tests { use crate::emit::pipeline::batcher::BatcherMsg; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; use crate::toast::MemChunkStore; - use ahash::{HashMap, HashMapExt}; + use ahash::{HashMap, HashMapExt, HashSetExt}; use walrus::pg::walparser::RelFileNode; fn rel(rel_node: u32) -> Arc { @@ -578,7 +567,7 @@ mod tests { mem_spool(), Default::default(), None, - std::collections::HashSet::new(), + HashSet::new(), )); let mut by_seq: HashMap = HashMap::new(); @@ -631,7 +620,7 @@ mod tests { mem_spool(), Default::default(), None, - std::collections::HashSet::new(), + HashSet::new(), )); let mut seqs: Vec = Vec::new(); @@ -681,7 +670,7 @@ mod tests { mem_spool(), Default::default(), None, - std::collections::HashSet::new(), + HashSet::new(), )); let mut rows = Vec::new(); @@ -747,7 +736,7 @@ mod tests { DeferredSpool::new(spool_tmp.path().join("bootstrap_deferred.bin"), 0), Default::default(), None, - std::collections::HashSet::new(), + HashSet::new(), )); let mut rows = Vec::new(); @@ -806,7 +795,7 @@ mod tests { mem_spool(), Default::default(), None, - std::collections::HashSet::new(), + HashSet::new(), )); // Wait for the referrer to defer, then unmap before walk EOF let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); diff --git a/src/emit/pipeline/decode.rs b/src/emit/pipeline/decode.rs index 511c7e03..06927a27 100644 --- a/src/emit/pipeline/decode.rs +++ b/src/emit/pipeline/decode.rs @@ -101,9 +101,10 @@ pub async fn decode_and_route( // One spool per xact; generations sealed before spooling carry None let spool = chunks.iter().find_map(|g| g.spool()); let mut routed = 0u64; - let mut buf: Vec = Vec::new(); + let mut heaps = heaps.into_iter(); + let mut buf: Vec = Vec::with_capacity(ctx.chunk_rows.min(heaps.len())); let mut buf_bytes = 0usize; - for envelope in heaps { + while let Some(envelope) = heaps.next() { // Discard precedes detoast: unrouted values never hit the resolver let Some(route) = envelope.route else { continue; @@ -143,6 +144,7 @@ pub async fn decode_and_route( routed += 1; if buf.len() >= ctx.chunk_rows || buf_bytes >= DECODE_CHUNK_BYTES { route_chunk(&ctx.msg_tx, std::mem::take(&mut buf), permit.clone()).await?; + buf.reserve(ctx.chunk_rows.min(heaps.len())); buf_bytes = 0; } } diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 9348da00..70d139b0 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -88,6 +88,70 @@ impl TailParts { } } +/// Tail plus the producer handles a single bootstrap leg holds +/// +/// The leg owns its seq space, so the durable watermark is throwaway: +/// completion is `wait_through(next_seq)`, not a resume position +pub struct OwnedTail { + pub msg_tx: mpsc::Sender, + pub ack: AckHandle, + parts: TailParts, + fatal: Fatal, + /// Prefix for the errors this leg reports + context: &'static str, +} + +impl OwnedTail { + /// Bounded legs pass `inserter_pool_size = 1`: they replay serially, and + /// the batcher's queue already overlaps insert with decode + pub async fn spawn( + emitter: &EmitterConfig, + inserter_pool_size: usize, + stats: Arc, + fatal: Fatal, + config_rx: Option>>, + oracle: Option>, + context: &'static str, + ) -> Result { + let (msg_tx, ack, parts) = spawn_with_config( + emitter, + inserter_pool_size, + stats, + Arc::new(Monotone::::new(0)), + fatal.clone(), + config_rx, + oracle, + ) + .await + .map_err(|e| format!("{context}: spawn insert tail: {e}"))?; + Ok(Self { + msg_tx, + ack, + parts, + fatal, + context, + }) + } + + /// Seal batches and prove every seq below `through` durable + pub async fn finish(self, through: u64) -> Result<(), String> { + self.parts + .finish(self.msg_tx, self.ack, through, &self.fatal) + .await + .map_err(|m| format!("{}: {m}", self.context)) + } + + /// Failed-leg teardown: dropping the last producer handles closes the + /// batcher, which final-flushes and cascades the inserters + collector + /// down. Bounded by the inserters' retry policy (a CH outage trips their + /// fatal, not a hang) + pub async fn quiesce(self) { + drop(self.msg_tx); + drop(self.ack); + self.parts.join().await; + } +} + /// Stand up the tail: ack collector, inserter pool (`n` connections), /// batcher. Returns the `BatcherMsg` sender + [`AckHandle`] (clone into /// producers) and join handles. Fails only if an inserter connection can't diff --git a/src/emit/route.rs b/src/emit/route.rs index a47e31fd..73d86bd7 100644 --- a/src/emit/route.rs +++ b/src/emit/route.rs @@ -77,6 +77,26 @@ impl RouteSnapshot { } } +/// One frozen route per mapped relation, so a bounded pass never re-reads +/// config or re-clones column rules per row +pub fn freeze_routes( + mapping: &crate::mapping::MappingSnapshot, + config: Option<&crate::config::ResolvedConfig>, + row_policy: &RowPolicy, +) -> ahash::HashMap> { + let rules = config.map_or_else(Arc::default, |rc| rc.column_rules.clone()); + mapping + .iter() + .map(|(name, m)| { + let policy = row_policy.for_rel(config, name); + ( + name.clone(), + RouteSnapshot::freeze(Arc::new(m.clone()), rules.clone(), policy), + ) + }) + .collect() +} + /// Described heap plus its resolved route. `route = None` means the relation /// is deterministically unmapped at that interval — a normal counted discard, /// distinct from a missing descriptor diff --git a/src/filter/dirty_tree.rs b/src/filter/dirty_tree.rs index 34a0a831..a8df85d2 100644 --- a/src/filter/dirty_tree.rs +++ b/src/filter/dirty_tree.rs @@ -139,7 +139,7 @@ impl DirtyTree { .map(|(x, _)| *x), ); let mut merged: Option = None; - let mut drained: Vec = Vec::new(); + let mut drained: Vec = Vec::with_capacity(members.len()); for x in members { if drained.contains(&x) { continue; diff --git a/src/filter/engine.rs b/src/filter/engine.rs index 3486d714..d9b238ba 100644 --- a/src/filter/engine.rs +++ b/src/filter/engine.rs @@ -441,7 +441,7 @@ impl Filter { // db 0 = shared relation; user rels there are impossible, kept for // symmetry with is_target_or_shared let mut capture_all = false; - let mut inval_oids: Vec = Vec::new(); + let mut inval_oids: Vec = Vec::with_capacity(payload.invals.relcache.len()); for inval in &payload.invals.relcache { if !self.is_target_or_shared(inval.db_id) { continue; @@ -526,7 +526,7 @@ impl Filter { let invals = parse_xact_invalidations(&record.main_data, page_magic)?; let namespace_hit = invals.namespace.hits(|db| self.is_target_or_shared(db)); let mut flush = false; - let mut oids: Vec = Vec::new(); + let mut oids: Vec = Vec::with_capacity(invals.relcache.len()); for inval in &invals.relcache { if !self.is_target_or_shared(inval.db_id) { continue; diff --git a/src/lib.rs b/src/lib.rs index eb087b8b..430e290e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,8 @@ pub mod xact; pub use backfill::{ backfill_bootstrap, backfill_staging, backfill_types, backup_backfill, backup_page_walk, backup_sentinel, backup_source, backup_source_direct, backup_source_object_store, - copy_backfill, opt_in, pg_path, spool, + bootstrap_window, copy_backfill, opt_in, pg_path, spool, visibility_gate, visibility_repair, + wal_replay, }; #[doc(hidden)] pub use catalog::{desc_log, pending, shadow, shadow_catalog, type_bridge}; diff --git a/src/ops/preflight.rs b/src/ops/preflight.rs index 3549fe95..5aa73231 100644 --- a/src/ops/preflight.rs +++ b/src/ops/preflight.rs @@ -17,6 +17,7 @@ //! - `--bootstrap-wal-from-archive` is set but source doesn't archive WAL: //! the bootstrap skips the inline WAL window and reads it from the //! `[backup]` bucket, so an unarchived source leaves nothing to fetch. +//! - source `max_wal_senders` cannot seat Direct backup and window stream use std::fmt; @@ -29,6 +30,9 @@ use crate::schema::RelName; /// Catalog accessors assume PG-16 column layouts; PG <16 unsupported. pub const MIN_SERVER_VERSION_NUM: i32 = 160_000; +/// Concurrent direct-backup replication connections +const DIRECT_BOOTSTRAP_SENDERS: i32 = 2; + #[derive(Debug, Error)] pub enum PreflightError { #[error( @@ -80,6 +84,13 @@ pub enum PreflightError { operator contract" )] ArchiveTargetEmpty, + #[error( + "source max_wal_senders={got} < {need}: a direct bootstrap holds {need} \ + replication connections at once — BASE_BACKUP, and the daemon's own, \ + which streams the backup window to the destination while the backup \ + runs. Raise max_wal_senders on the source" + )] + WalSendersTooLow { got: i32, need: i32 }, #[error("pg query: {0}")] Pg(#[from] tokio_postgres::Error), #[error("shadow_version_num could not be parsed: {0:?}")] @@ -142,6 +153,8 @@ pub struct ShadowInputs<'a> { /// Checks that gate `BASE_BACKUP`, run before `run_bootstrap`. pub struct BootstrapInputs<'a> { pub source_sql: &'a Client, + /// Stream window live instead of replaying hydrated segments + pub window_leg: bool, /// Bootstrap will set `wal: false` and hydrate shadow's `pg_wal/` from /// the `[backup]` bucket instead of from `base.tar`. pub wal_from_archive: bool, @@ -267,6 +280,19 @@ pub async fn shadow(input: ShadowInputs<'_>) -> Result) -> Result { let mut report = PreflightReport { errors: Vec::new() }; + if input.window_leg { + let got: i32 = input + .source_sql + .query_one("SELECT current_setting('max_wal_senders')::int", &[]) + .await? + .get(0); + if got < DIRECT_BOOTSTRAP_SENDERS { + report.errors.push(PreflightError::WalSendersTooLow { + got, + need: DIRECT_BOOTSTRAP_SENDERS, + }); + } + } if !input.wal_from_archive { return Ok(report); } diff --git a/src/record.rs b/src/record.rs index 54b8312f..aeb93d8f 100644 --- a/src/record.rs +++ b/src/record.rs @@ -13,6 +13,30 @@ use crate::filter::manifest::Manifest; pub const WAL_SEG_SIZE: u64 = walrus::pg::wal::segment::DEFAULT_WAL_SEG_SIZE; +/// Complete segments covering half-open `range` on `timeline`. A boundary +/// `range.end` needs no further segment; inclusive callers pass +/// `end.saturating_add(1)` +pub fn segments_covering(timeline: u32, range: std::ops::Range) -> Vec { + let mut cur = SegmentName { + timeline, + log_id: (range.start >> 32) as u32, + seg_no: ((range.start & 0xFFFF_FFFF) / WAL_SEG_SIZE) as u32, + }; + let segment_count = range + .end + .saturating_sub(cur.start_lsn(WAL_SEG_SIZE)) + .div_ceil(WAL_SEG_SIZE) + .max(1); + let mut out = Vec::with_capacity(segment_count as usize); + loop { + out.push(cur); + if range.end <= cur.start_lsn(WAL_SEG_SIZE).saturating_add(WAL_SEG_SIZE) { + break out; + } + cur = cur.next(WAL_SEG_SIZE); + } +} + /// Numeric id fallback for unknown rmgrs pub fn rmgr_label(rm: u8) -> String { let named = match rm { diff --git a/src/schema.rs b/src/schema.rs index 968eac23..ef4181c6 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -87,6 +87,44 @@ pub enum ReplIdent { }, } +impl ReplIdent { + /// `pg_class.relreplident` + pub fn to_char(&self) -> char { + match self { + ReplIdent::Default { .. } => 'd', + ReplIdent::Nothing => 'n', + ReplIdent::Full { .. } => 'f', + ReplIdent::UsingIndex { .. } => 'i', + } + } + + /// Build from `pg_class.relreplident` plus the `pg_index` rows it names: + /// the primary key for `d`/`f`, the replica-identity index for `i` + pub fn from_parts( + c: char, + pk_attnums: Option>, + using_index: Option<(Oid, Vec)>, + ) -> Result { + match c { + 'd' => Ok(ReplIdent::Default { pk_attnums }), + 'n' => Ok(ReplIdent::Nothing), + 'f' => Ok(ReplIdent::Full { pk_attnums }), + 'i' => { + let (index_oid, key_attnums) = using_index.ok_or_else(|| { + "relreplident='i' but no pg_index row with indisreplident=true".to_owned() + })?; + Ok(ReplIdent::UsingIndex { + index_oid, + key_attnums, + }) + } + other => Err(format!( + "unknown relreplident {other:?} (expected one of d/n/f/i)" + )), + } + } +} + /// Resolve stored primary/index keys, including primary key metadata under `Full` pub fn replident_key_attnums(desc: &RelDescriptor) -> &[i16] { match &desc.replident { diff --git a/src/source/catalog_capture.rs b/src/source/catalog_capture.rs index 1af42a22..ae1be79a 100644 --- a/src/source/catalog_capture.rs +++ b/src/source/catalog_capture.rs @@ -305,13 +305,13 @@ impl CatalogCapture { /// shape after them, so the oid's last non-`Retired` entry is the shape /// CH is told about. fn replay_events(&self, batch: &BatchRecord) -> Vec { - let mut last_for_oid: HashMap = HashMap::new(); + let mut last_for_oid: HashMap = HashMap::with_capacity(batch.entries.len()); for (at, entry) in batch.entries.iter().enumerate() { if !matches!(entry.value, LogValue::Retired) { last_for_oid.insert(entry.oid, at); } } - let mut out = Vec::new(); + let mut out = Vec::with_capacity(last_for_oid.len()); for (at, entry) in batch.entries.iter().enumerate() { if last_for_oid.get(&entry.oid) != Some(&at) { continue; diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index d59f1685..9e8115f8 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -575,13 +575,14 @@ impl ClickHouseChunkStore { ) } - /// Bloom-prune candidate TIDs, then aggregate full history before filtering + /// Prune candidates, then break reused-generation ties by TID fn fetch_sql(&self, toast_relid: u32, value_id: u32, max_lsn: u64) -> String { let table = self.toast_table(toast_relid); format!( - "SELECT `chunk_seq`, argMax(`chunk_data`, `ver`) AS `chunk_data`\n\ + "SELECT `chunk_seq`, argMax(`chunk_data`, (`ver`, `blkno`, `offnum`)) AS `chunk_data`\n\ FROM (\n \ - SELECT argMax(`chunk_id`, `_lsn`) AS `chunk_id`,\n \ + SELECT `blkno`, `offnum`,\n \ + argMax(`chunk_id`, `_lsn`) AS `chunk_id`,\n \ argMax(`chunk_seq`, `_lsn`) AS `chunk_seq`,\n \ argMax(`chunk_data`, `_lsn`) AS `chunk_data`,\n \ max(`_lsn`) AS `ver`,\n \ @@ -655,7 +656,8 @@ impl ClickHouseChunkStore { let mut lsn = Vec::with_capacity(n * 8); let mut is_deleted = Vec::with_capacity(n); let mut offsets = Vec::with_capacity(n); - let mut data = Vec::new(); + let data_len = group.iter().map(|r| r.chunk_data.len()).sum(); + let mut data = Vec::with_capacity(data_len); for r in &group { blkno.extend_from_slice(&r.blkno.to_le_bytes()); offnum.extend_from_slice(&r.offnum.to_le_bytes()); @@ -1572,9 +1574,10 @@ mod tests { assert_eq!( store.fetch_sql(16500, 7, 0x2000), - "SELECT `chunk_seq`, argMax(`chunk_data`, `ver`) AS `chunk_data`\n\ + "SELECT `chunk_seq`, argMax(`chunk_data`, (`ver`, `blkno`, `offnum`)) AS `chunk_data`\n\ FROM (\n \ - SELECT argMax(`chunk_id`, `_lsn`) AS `chunk_id`,\n \ + SELECT `blkno`, `offnum`,\n \ + argMax(`chunk_id`, `_lsn`) AS `chunk_id`,\n \ argMax(`chunk_seq`, `_lsn`) AS `chunk_seq`,\n \ argMax(`chunk_data`, `_lsn`) AS `chunk_data`,\n \ max(`_lsn`) AS `ver`,\n \ diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index a4c4b3f8..3d974534 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -36,7 +36,7 @@ //! Spill-to-ClickHouse (Option B) is deferred; v1 is local-disk-only. use std::cmp::Reverse; -use std::collections::{BinaryHeap, VecDeque}; +use std::collections::{BTreeSet, BinaryHeap, VecDeque, hash_map::Entry}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -613,8 +613,8 @@ pub async fn resolve_stash( if rfns.is_empty() { return Ok(()); } - let mut outcomes: HashMap = HashMap::new(); - let mut barriers: Vec<(u32, u64)> = Vec::new(); + let mut outcomes: HashMap = HashMap::with_capacity(rfns.len()); + let mut barriers: Vec<(u32, u64)> = Vec::with_capacity(rfns.len()); for (rfn, mark) in &rfns { let (rfn, mark) = (*rfn, *mark); match log.descriptor_at_spanned(rfn, next_lsn) { @@ -870,8 +870,7 @@ impl XactBuffer { .iter() .map(|(xid, st)| { let mut last_lsn = st.first_lsn.get(); - let mut rels: std::collections::BTreeSet<(u32, u32)> = - std::collections::BTreeSet::new(); + let mut rels: BTreeSet<(u32, u32)> = BTreeSet::new(); let mut heap_count = 0u64; let mut chunk_count = 0u64; for e in &st.in_mem { @@ -1785,10 +1784,10 @@ impl MergedDrain { self.chunk_bytes += mem_len + CHUNK_REF_META; self.chunk_gauge.add(mem_len + CHUNK_REF_META); match self.chunks.entry((c.toast_relid, c.value_id)) { - std::collections::hash_map::Entry::Occupied(mut o) => { + Entry::Occupied(mut o) => { o.get_mut().push(c.chunk_seq, body); } - std::collections::hash_map::Entry::Vacant(v) => { + Entry::Vacant(v) => { v.insert(ValueRef::new(c.chunk_seq, body)); } } @@ -2324,7 +2323,7 @@ pub async fn detoast_heap( // Attached at decode: same descriptor interpretation from decode to // detoast regardless of captures landing in between let rel = heap.descriptor.clone(); - let mut uses: HashMap<(u32, u32), u32> = HashMap::new(); + let mut uses: HashMap<(u32, u32), u32> = HashMap::with_capacity(pointers.len()); for p in &pointers { *uses.entry((p.va_toastrelid, p.va_valueid)).or_default() += 1; } @@ -3832,10 +3831,10 @@ mod tests { for &(seq, body) in chunks { let body = Body::Mem(bytes::Bytes::from_static(body)); match map.entry(key) { - std::collections::hash_map::Entry::Occupied(mut o) => { + Entry::Occupied(mut o) => { o.get_mut().push(seq, body); } - std::collections::hash_map::Entry::Vacant(v) => { + Entry::Vacant(v) => { v.insert(ValueRef::new(seq, body)); } } @@ -3888,10 +3887,10 @@ mod tests { for &(seq, body) in chunks { let r = Body::File(w.append(body).unwrap()); match map.entry(key) { - std::collections::hash_map::Entry::Occupied(mut o) => { + Entry::Occupied(mut o) => { o.get_mut().push(seq, r); } - std::collections::hash_map::Entry::Vacant(v) => { + Entry::Vacant(v) => { v.insert(ValueRef::new(seq, r)); } } diff --git a/tests/add_column_default.rs b/tests/add_column_default.rs index 0f9d59ba..f564ea0f 100644 --- a/tests/add_column_default.rs +++ b/tests/add_column_default.rs @@ -30,16 +30,7 @@ use walshadow::schema::RelName; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn add_column_default_replicates_pre_alter_default() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } diff --git a/tests/bootstrap_crossing_xact_ch.rs b/tests/bootstrap_crossing_xact_ch.rs new file mode 100644 index 00000000..85825259 --- /dev/null +++ b/tests/bootstrap_crossing_xact_ch.rs @@ -0,0 +1,143 @@ +//! Verify pump rebuilds transaction open across handoff from +//! window leg's `open_floor` + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use walshadow::mapping::TableTarget; +use walshadow::schema::RelName; +use walshadow::source_feed::open_sql_client; + +const N_ROWS: i32 = 64; +/// Keep backup window open for writer +const MAX_RATE_KIB: &str = "32768"; +const SCHEMA: &str = "s21"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn transaction_open_across_the_handoff_reaches_ch() { + crossing_transaction(true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn transaction_open_across_the_handoff_reaches_ch_without_slot() { + crossing_transaction(false).await; +} + +async fn crossing_transaction(with_slot: bool) { + if !fx::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + + fx::load_source_workload(&source, SCHEMA, N_ROWS).expect("load source workload"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + fx::create_ch_dest_table(&ch, "default", "t").expect("create ch table"); + + let ch_config_path = tmp.path().join("ch-config.toml"); + fx::write_ch_config_toml( + &ch_config_path, + "127.0.0.1", + slot.ch_tcp, + "default", + &RelName::new(SCHEMA, "t"), + &TableTarget::new("default", "t"), + ) + .expect("write ch-config"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let mut args = vec!["--bootstrap-max-rate-kib", MAX_RATE_KIB]; + if with_slot { + args.extend(["--slot", "walshadow_crossing"]); + } + let child = daemon + .spawn(&source, &ch_config_path, slot.walsender, &args) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result: Result<()> = async { + fx::wait_for_backup_streaming(&source, Duration::from_secs(60))?; + + // Hold transaction across handoff + let holder = open_sql_client(&fx::pg_cfg(&source, "crossing-xact-test")) + .await + .context("holder connect")?; + holder.batch_execute("BEGIN").await.context("BEGIN")?; + holder + .batch_execute(&format!( + "INSERT INTO {SCHEMA}.t SELECT g, 'crossing-'||g::text \ + FROM generate_series(1001, 1100) g" + )) + .await + .context("crossing INSERT")?; + // Move default resume beyond transaction prefix + source + .apply_schema_dump("SELECT pg_switch_wal();\nSELECT pg_switch_wal();\n") + .context("switch wal")?; + + // Advance end_lsn beyond held records + let mut round = 0; + while fx::backup_in_progress(&source) { + round += 1; + source + .apply_schema_dump(&format!( + "INSERT INTO {SCHEMA}.t SELECT g, 'late-'||g::text \ + FROM generate_series({from}, {to}) g;\n", + from = 2000 + round * 10, + to = 2009 + round * 10, + )) + .context("in-window writes")?; + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // Direct mode can fall back to landed WAL + daemon.wait_for_log("backup window shipped", Duration::from_secs(90))?; + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(60)) + .context("daemon metrics endpoint never came up")?; + + holder.batch_execute("COMMIT").await.context("COMMIT")?; + + let src_count = source + .psql_one(&format!("SELECT count(*) FROM {SCHEMA}.t")) + .context("source count")?; + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &src_count, + Duration::from_secs(60), + ) + .context("crossing rows never reached CH")?; + fx::assert_ch_matches_source(&ch, &source, &format!("{SCHEMA}.t"), "default.t") + .context("source vs CH parity across the handoff")?; + let crossing = ch + .query("SELECT count() FROM default.t FINAL WHERE id BETWEEN 1001 AND 1100") + .context("crossing count")?; + ensure!(crossing == "100", "crossing batch incomplete: {crossing}"); + + // Require crossing transaction at window seal + let log = daemon.stderr(); + let shipped = log + .lines() + .find(|l| l.contains("backup window shipped")) + .context("daemon logged no window-leg summary")?; + ensure!( + shipped.contains("open_floor=\""), + "window leg reported no open xact: {shipped}", + ); + Ok(()) + } + .await; + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_direct_ch.rs b/tests/bootstrap_direct_ch.rs index a4e89f89..fe47920f 100644 --- a/tests/bootstrap_direct_ch.rs +++ b/tests/bootstrap_direct_ch.rs @@ -32,44 +32,17 @@ #[path = "common/bootstrap_ch_fixture.rs"] mod fx; -use std::fs; -use std::net::SocketAddr; -use std::os::unix::process::CommandExt; -use std::process::{Command, Stdio}; use std::time::Duration; use anyhow::{Context, Result}; use walshadow::mapping::TableTarget; use walshadow::schema::RelName; -use walshadow::shadow::{Shadow, ShadowConfig}; const N_ROWS: i32 = 64; -fn make_source(tmp: &tempfile::TempDir) -> Shadow { - let mut cfg = ShadowConfig::new( - tmp.path().join("source-data"), - tmp.path().join("source-filtered"), - ); - cfg.port = fx::PG_SOURCE_PORT; - cfg.socket_dir = tmp.path().join("source-sock"); - cfg.ctl_timeout = Duration::from_secs(60); - fs::create_dir_all(&cfg.filter_out_dir).unwrap(); - fs::create_dir_all(&cfg.socket_dir).unwrap(); - Shadow::new(cfg) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn direct_bootstrap_ch_end_to_end() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } @@ -77,11 +50,7 @@ async fn direct_bootstrap_ch_end_to_end() { let tmp = tempfile::tempdir().unwrap(); // 1. Source PG. - let source = make_source(&tmp); - source.initdb().expect("initdb source"); - source.write_base_conf().expect("source base conf"); - fx::append_source_conf(&source).expect("append source conf"); - source.start().expect("start source"); + let source = fx::start_source(&tmp); let _src_stop = fx::StopOnDrop { sh: &source }; // 2. Source schema + workload (64 rows). @@ -106,103 +75,30 @@ async fn direct_bootstrap_ch_end_to_end() { // 5. Shadow data dir and socket layout. Daemon writes listener // config and sets data dir mode to 0700 before pg_ctl start - let bootstrap_shadow_data_dir = tmp.path().join("shadow-data"); - let shadow_sock = tmp.path().join("shadow-sock"); - fs::create_dir_all(&shadow_sock).unwrap(); - let shadow_filter_dir = tmp.path().join("filtered"); - fs::create_dir_all(&shadow_filter_dir).unwrap(); - let spill_dir = tmp.path().join("spill"); - fs::create_dir_all(&spill_dir).unwrap(); - - // 6. Spawn walshadow-stream subprocess. `--max-segments=2` so the - // daemon ships the bootstrap-induced segment-3 transition plus - // one more (the post-workload `pg_switch_wal` below) before - // exiting. Setting `=1` races: the bootstrap pump consumes the - // first segment shipment before `wait_for_listen` polls again, - // so the test never catches a listening metrics port. - // `--metrics-bind` doubles as a "bootstrap complete + shadow up - // + WAL pump running" readiness probe. - let bin = env!("CARGO_BIN_EXE_walshadow-stream"); - let stderr_path = tmp.path().join("daemon.stderr.log"); - let stderr_file = fs::File::create(&stderr_path).expect("open daemon stderr log"); - let metrics_addr: SocketAddr = format!("127.0.0.1:{}", slot.metrics).parse().unwrap(); - let child = Command::new(bin) - .args([ - "--host", - source.config().socket_dir.to_str().unwrap(), - "--port", - &fx::PG_SOURCE_PORT.to_string(), - "--user", - "postgres", - "--dbname", - "postgres", - "--sslmode", - "disable", - "--out-dir", - shadow_filter_dir.to_str().unwrap(), - "--shadow-socket-dir", - shadow_sock.to_str().unwrap(), - "--shadow-port", - &fx::PG_SHADOW_PORT.to_string(), - "--shadow-user", - "postgres", - "--shadow-dbname", - "postgres", - "--spill-dir", - spill_dir.to_str().unwrap(), - "--status-interval", - "1", - "--metrics-bind", - &metrics_addr.to_string(), - "--walsender-bind", - &format!("127.0.0.1:{}", slot.walsender), - "--retention-bytes", - "0", - "--ch-config", - ch_config_path.to_str().unwrap(), - "--bootstrap-mode", - "direct", - "--bootstrap-shadow-data-dir", - bootstrap_shadow_data_dir.to_str().unwrap(), - "--bootstrap-shadow-replay-timeout", - "120", - ]) - .env("RUST_LOG", "warn,walshadow=info") - .stdout(Stdio::null()) - .stderr(Stdio::from(stderr_file)) - .process_group(0) - .spawn() + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn(&source, &ch_config_path, slot.walsender, &[]) .expect("spawn walshadow-stream"); let guard = fx::ChildGuard::new(child); let result = (|| -> Result<()> { - // 7. Wait for the daemon's metrics endpoint (liveness). The daemon + // 6. Wait for the daemon's metrics endpoint (liveness). The daemon // binds it before the bootstrap drains to CH, so this is not a // bootstrap-complete signal on its own. - fx::wait_for_listen(metrics_addr, Duration::from_secs(30)) + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(30)) .context("daemon metrics endpoint never came up")?; - // 8. Poll until the bootstrap rows are durable on CH — the tail - // drains asynchronously, so racing it with an immediate assert - // flakes on slow CI. let src_count = source .psql_one("SELECT count(*) FROM s14.t") .context("source count")?; - let deadline = std::time::Instant::now() + Duration::from_secs(60); - loop { - let n = ch - .query("SELECT count() FROM default.t FINAL WHERE _is_deleted = 0") - .unwrap_or_default(); - if n == src_count { - break; - } - if std::time::Instant::now() >= deadline { - anyhow::bail!("bootstrap rows never reached CH: source={src_count}, ch={n}"); - } - std::thread::sleep(Duration::from_millis(200)); - } - - // 9. Oracle: count + sum(id) + md5(string_agg(name, ',' ORDER BY id)) + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &src_count, + Duration::from_secs(60), + )?; + + // 7. Oracle: count + sum(id) + md5(string_agg(name, ',' ORDER BY id)) // must match across both sides. fx::assert_ch_matches_source(&ch, &source, "s14.t", "default.t") .context("source vs CH parity")?; @@ -210,24 +106,5 @@ async fn direct_bootstrap_ch_end_to_end() { Ok(()) })(); - // 11. Kill daemon before shadow so supervisor cannot restart it - // SIGKILL skips shadow cleanup, stop any remaining postmaster - let _ = guard.into_inner().map(|mut c| { - let _ = c.kill(); - let _ = c.wait(); - }); - if bootstrap_shadow_data_dir.join("postmaster.pid").exists() { - let mut shadow_cfg = - ShadowConfig::new(bootstrap_shadow_data_dir.clone(), shadow_filter_dir.clone()); - shadow_cfg.port = fx::PG_SHADOW_PORT; - shadow_cfg.socket_dir = shadow_sock.clone(); - shadow_cfg.ctl_timeout = Duration::from_secs(60); - let shadow = Shadow::new(shadow_cfg); - let _ = shadow.stop(); - } - - if let Err(e) = result { - let stderr = fs::read_to_string(&stderr_path).unwrap_or_default(); - panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); - } + fx::finish_daemon(guard, &daemon, result); } diff --git a/tests/bootstrap_gate_ch.rs b/tests/bootstrap_gate_ch.rs new file mode 100644 index 00000000..4cd854ae --- /dev/null +++ b/tests/bootstrap_gate_ch.rs @@ -0,0 +1,157 @@ +//! Verify greenfield gate excludes dead and aborted fixed-width tuples + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::fs; +use std::path::Path; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use walshadow::shadow::Shadow; + +const N_ROWS: i32 = 64; +/// Rows surviving DELETE +const N_LIVE: i32 = 32; +/// Dead and aborted tuples +const N_GATED: u64 = 64; + +/// Leave dead and aborted tuples unpruned for backup +fn load_gated_workload(source: &Shadow, schema: &str) -> Result<()> { + let sql = format!( + "CREATE SCHEMA {schema};\n\ + CREATE TABLE {schema}.t (id int4 PRIMARY KEY, n int8 NOT NULL) \ + WITH (autovacuum_enabled = false);\n\ + ALTER TABLE {schema}.t REPLICA IDENTITY FULL;\n\ + INSERT INTO {schema}.t \ + SELECT g, g * 10 FROM generate_series(1, {N_ROWS}) g;\n\ + DELETE FROM {schema}.t WHERE id % 2 = 0;\n\ + BEGIN;\n\ + INSERT INTO {schema}.t \ + SELECT g, g * 10 FROM generate_series(1001, 1032) g;\n\ + ROLLBACK;\n\ + CHECKPOINT;\n\ + SELECT pg_switch_wal();\n", + ); + source.apply_schema_dump(&sql)?; + Ok(()) +} + +/// Keep relation on page-walk path with fixed-width mapping +fn write_gate_ch_config(path: &Path, ch_port: u16, schema: &str) -> Result<()> { + let body = format!( + "[ch]\n\ + host = \"127.0.0.1\"\n\ + port = {ch_port}\n\ + database = \"default\"\n\ + compression = \"lz4\"\n\ + \n\ + [table.\"{schema}\".\"t\"]\n\ + target_database = \"default\"\n\ + target_table = \"t\"\n\ + columns = [\n \ + {{ attnum = 1, target = \"id\", type = \"Int32\" }},\n \ + {{ attnum = 2, target = \"n\", type = \"Int64\" }},\n\ + ]\n", + ); + fs::write(path, body).with_context(|| format!("write ch-config {}", path.display()))?; + Ok(()) +} + +fn create_ch_dest_table(ch: &fx::ChServer) -> Result<()> { + ch.query("CREATE DATABASE IF NOT EXISTS default")?; + ch.query( + "CREATE OR REPLACE TABLE default.t (\ + id Int32,\ + n Int64,\ + _lsn UInt64,\ + _xid UInt32,\ + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool\ + ) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + )?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn dead_and_aborted_tuples_stay_out_of_ch() { + if !fx::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + + load_gated_workload(&source, "s17").expect("load gated workload"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + create_ch_dest_table(&ch).expect("create ch table"); + + let ch_config_path = tmp.path().join("ch-config.toml"); + write_gate_ch_config(&ch_config_path, slot.ch_tcp, "s17").expect("write ch-config"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn(&source, &ch_config_path, slot.walsender, &[]) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result = (|| -> Result<()> { + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(30)) + .context("daemon metrics endpoint never came up")?; + + // Live tuples ship during the walk, so CH parity runs ahead of the + // gate. Wait on its verdict before reading either + daemon + .wait_for_log("bootstrap visibility gate settled", Duration::from_secs(60)) + .context("gate never logged its verdict")?; + + // Hold at live count to catch later ghost rows + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &N_LIVE.to_string(), + Duration::from_secs(60), + )?; + // Ghost rows carry ids the source never committed + let ghosts = ch + .query("SELECT count() FROM default.t FINAL WHERE id >= 1000") + .context("ghost count")?; + ensure!(ghosts == "0", "aborted rows reached CH: {ghosts}"); + let src_sum = source + .psql_one("SELECT coalesce(sum(n), 0)::text FROM s17.t") + .context("source sum")?; + let ch_sum = ch + .query("SELECT sum(n) FROM default.t FINAL WHERE _is_deleted = 0") + .context("ch sum")?; + ensure!( + src_sum == ch_sum, + "sum(n) differs: ch={ch_sum} src={src_sum}" + ); + + // Dead rows require backup pg_xact verdict + let stderr = daemon.stderr(); + let line = stderr + .lines() + .find(|l| l.contains("bootstrap visibility gate settled")) + .context("gate verdict left the log")?; + ensure!( + line.contains(&format!("gated={N_GATED}")), + "gate verdict off: {line}" + ); + ensure!(!line.contains("deferred=0"), "nothing deferred: {line}"); + // Fixed-width relation bypasses repair + ensure!( + line.contains("pending_relations=0"), + "relation left the walk: {line}" + ); + Ok(()) + })(); + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_object_store_ch.rs b/tests/bootstrap_object_store_ch.rs index ab3af596..69688a68 100644 --- a/tests/bootstrap_object_store_ch.rs +++ b/tests/bootstrap_object_store_ch.rs @@ -33,10 +33,7 @@ mod fx; use std::fs; -use std::net::SocketAddr; -use std::os::unix::process::CommandExt; use std::path::PathBuf; -use std::process::{Command, Stdio}; use std::sync::Arc; use std::time::Duration; @@ -51,7 +48,7 @@ use walrus::storage::DynStorage; use walrus::storage::fs::FsStorage; use walshadow::mapping::TableTarget; use walshadow::schema::RelName; -use walshadow::shadow::{Shadow, ShadowConfig}; +use walshadow::shadow::Shadow; const N_ROWS: i32 = 64; @@ -83,19 +80,6 @@ async fn push_completed_wal_segments( Ok(()) } -fn make_source(tmp: &tempfile::TempDir) -> Shadow { - let mut cfg = ShadowConfig::new( - tmp.path().join("source-data"), - tmp.path().join("source-filtered"), - ); - cfg.port = fx::PG_SOURCE_PORT; - cfg.socket_dir = tmp.path().join("source-sock"); - cfg.ctl_timeout = Duration::from_secs(60); - fs::create_dir_all(&cfg.filter_out_dir).unwrap(); - fs::create_dir_all(&cfg.socket_dir).unwrap(); - Shadow::new(cfg) -} - /// Minimal Settings for an uncompressed `FsStorage` root — matches /// `bootstrap_object_store_e2e.rs::test_settings`. fn test_settings(storage_root: PathBuf) -> Settings { @@ -111,12 +95,7 @@ fn test_settings(storage_root: PathBuf) -> Settings { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn object_store_bootstrap_ch_end_to_end() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } @@ -124,11 +103,7 @@ async fn object_store_bootstrap_ch_end_to_end() { let tmp = tempfile::tempdir().unwrap(); // 1. Source PG. - let source = make_source(&tmp); - source.initdb().expect("initdb source"); - source.write_base_conf().expect("source base conf"); - fx::append_source_conf(&source).expect("append source conf"); - source.start().expect("start source"); + let source = fx::start_source(&tmp); let _src_stop = fx::StopOnDrop { sh: &source }; // 2. Schema + workload. @@ -205,132 +180,51 @@ async fn object_store_bootstrap_ch_end_to_end() { ch_config_body.push_str(&format!("\n[backup]\narchive = \"{archive_uri}\"\n")); fs::write(&ch_config_path, ch_config_body).expect("append [backup] to ch-config"); - // 6. Shadow layout. Daemon writes port and socket config, so test - // does not add source settings before bootstrap - let bootstrap_shadow_data_dir = tmp.path().join("shadow-data"); - let shadow_sock = tmp.path().join("shadow-sock"); - fs::create_dir_all(&shadow_sock).unwrap(); - let shadow_filter_dir = tmp.path().join("filtered"); - fs::create_dir_all(&shadow_filter_dir).unwrap(); - let spill_dir = tmp.path().join("spill"); - fs::create_dir_all(&spill_dir).unwrap(); - - // 7. Spawn walshadow-stream. The daemon reads the archive from the + // 6. Spawn walshadow-stream. The daemon reads the archive from the // `[backup]` section of `--ch-config` (built into a // `walrus::config::Settings`), not `WALG_*` env. - let bin = env!("CARGO_BIN_EXE_walshadow-stream"); - let stderr_path = tmp.path().join("daemon.stderr.log"); - let stderr_file = fs::File::create(&stderr_path).expect("open daemon stderr log"); - let metrics_addr: SocketAddr = format!("127.0.0.1:{}", slot.metrics).parse().unwrap(); - let child = Command::new(bin) - .args([ - "--host", - source.config().socket_dir.to_str().unwrap(), - "--port", - &fx::PG_SOURCE_PORT.to_string(), - "--user", - "postgres", - "--dbname", - "postgres", - "--sslmode", - "disable", - "--out-dir", - shadow_filter_dir.to_str().unwrap(), - "--shadow-socket-dir", - shadow_sock.to_str().unwrap(), - "--shadow-port", - &fx::PG_SHADOW_PORT.to_string(), - "--shadow-user", - "postgres", - "--shadow-dbname", - "postgres", - "--spill-dir", - spill_dir.to_str().unwrap(), - "--status-interval", - "1", - "--metrics-bind", - &metrics_addr.to_string(), - "--walsender-bind", - &format!("127.0.0.1:{}", slot.walsender), - "--retention-bytes", - "0", - "--ch-config", - ch_config_path.to_str().unwrap(), - "--bootstrap-mode", + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn_mode( + &source, + &ch_config_path, + slot.walsender, "object-store", - "--bootstrap-shadow-data-dir", - bootstrap_shadow_data_dir.to_str().unwrap(), - "--bootstrap-backup-name", - &backup_name, - "--bootstrap-shadow-replay-timeout", - "120", - ]) - .env("PGHOST", source.config().socket_dir.to_str().unwrap()) - .env("PGPORT", fx::PG_SOURCE_PORT.to_string()) - .env("PGUSER", "postgres") - .env("PGDATABASE", "postgres") - .env("RUST_LOG", "warn,walshadow=info") - .stdout(Stdio::null()) - .stderr(Stdio::from(stderr_file)) - .process_group(0) - .spawn() + &["--bootstrap-backup-name", &backup_name], + &[ + ("PGHOST", socket_host.clone()), + ("PGPORT", source.config().port.to_string()), + ("PGUSER", "postgres".into()), + ("PGDATABASE", "postgres".into()), + ], + ) .expect("spawn walshadow-stream"); let guard = fx::ChildGuard::new(child); let result = (|| -> Result<()> { - // 8. Wait for the daemon's metrics endpoint (liveness). The daemon + // 7. Wait for the daemon's metrics endpoint (liveness). The daemon // binds it before the bootstrap tail drains to CH, so it is not // a bootstrap-complete signal on its own. - fx::wait_for_listen(metrics_addr, Duration::from_secs(30)) + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(30)) .context("daemon metrics endpoint never came up")?; - // 9. Poll until the bootstrap rows are durable on CH — the tail - // drains asynchronously, so racing it with an immediate assert - // flakes on slow CI. let src_count = source .psql_one("SELECT count(*) FROM s14.t") .context("source count")?; - let deadline = std::time::Instant::now() + Duration::from_secs(60); - loop { - let n = ch - .query("SELECT count() FROM default.t FINAL WHERE _is_deleted = 0") - .unwrap_or_default(); - if n == src_count { - break; - } - if std::time::Instant::now() >= deadline { - anyhow::bail!("bootstrap rows never reached CH: source={src_count}, ch={n}"); - } - std::thread::sleep(Duration::from_millis(200)); - } - - // 10. Oracle. ChildGuard's Drop SIGKILLs the daemon at end of scope; - // no `pg_switch_wal` + drain cycle since the surface is bootstrap - // correctness, not streaming. + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &src_count, + Duration::from_secs(60), + )?; + + // 8. Oracle. No `pg_switch_wal` + drain cycle since the surface is + // bootstrap correctness, not streaming. fx::assert_ch_matches_source(&ch, &source, "s14.t", "default.t") .context("source vs CH parity")?; Ok(()) })(); - // 12. Kill daemon before shadow so supervisor cannot restart it - // Stop any remaining postmaster - let _ = guard.into_inner().map(|mut c| { - let _ = c.kill(); - let _ = c.wait(); - }); - if bootstrap_shadow_data_dir.join("postmaster.pid").exists() { - let mut shadow_cfg = - ShadowConfig::new(bootstrap_shadow_data_dir.clone(), shadow_filter_dir.clone()); - shadow_cfg.port = fx::PG_SHADOW_PORT; - shadow_cfg.socket_dir = shadow_sock.clone(); - shadow_cfg.ctl_timeout = Duration::from_secs(60); - let shadow = Shadow::new(shadow_cfg); - let _ = shadow.stop(); - } - - if let Err(e) = result { - let stderr = fs::read_to_string(&stderr_path).unwrap_or_default(); - panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); - } + fx::finish_daemon(guard, &daemon, result); } diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 465622b8..759d9c37 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -183,7 +183,7 @@ async fn bootstrap_tail_fans_out_n2() { ), Default::default(), None, - std::collections::HashSet::new(), + ahash::HashSet::default(), )); let outcome = drain.await.expect("drain join").expect("drain ok"); assert_eq!(outcome.next_seq, 2, "one seq per rfn"); diff --git a/tests/bootstrap_toast_gate_ch.rs b/tests/bootstrap_toast_gate_ch.rs new file mode 100644 index 00000000..0ee71919 --- /dev/null +++ b/tests/bootstrap_toast_gate_ch.rs @@ -0,0 +1,217 @@ +//! Verify greenfield repairs TOAST-owning relations and seeds chunk mirror +//! for later unchanged external pointers + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::fs; +use std::path::Path; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail, ensure}; +use walshadow::shadow::Shadow; + +/// Force multi-chunk external values +const BODY_REPEAT: u32 = 700; +const N_LIVE: i32 = 4; + +/// Leave live, dead, superseded, and aborted values unpruned +fn load_toast_workload(source: &Shadow, schema: &str) -> Result<()> { + let sql = format!( + "CREATE SCHEMA {schema};\n\ + CREATE TABLE {schema}.t (id int4 PRIMARY KEY, body text NOT NULL) \ + WITH (autovacuum_enabled = false);\n\ + ALTER TABLE {schema}.t ALTER COLUMN body SET STORAGE EXTERNAL;\n\ + ALTER TABLE {schema}.t REPLICA IDENTITY FULL;\n\ + INSERT INTO {schema}.t \ + SELECT g, repeat('live-'||g::text||'---', {BODY_REPEAT}) \ + FROM generate_series(1, {N_LIVE}) g;\n\ + INSERT INTO {schema}.t \ + SELECT g, repeat('dead-'||g::text||'---', {BODY_REPEAT}) \ + FROM generate_series(101, 104) g;\n\ + DELETE FROM {schema}.t WHERE id BETWEEN 101 AND 104;\n\ + UPDATE {schema}.t SET body = repeat('fresh-1---', {BODY_REPEAT}) WHERE id = 1;\n\ + BEGIN;\n\ + INSERT INTO {schema}.t \ + SELECT g, repeat('ghost-'||g::text||'---', {BODY_REPEAT}) \ + FROM generate_series(1001, 1004) g;\n\ + ROLLBACK;\n\ + CHECKPOINT;\n\ + SELECT pg_switch_wal();\n", + ); + source.apply_schema_dump(&sql)?; + Ok(()) +} + +/// Configure body mapping and ClickHouse chunk store +fn write_toast_ch_config(path: &Path, ch_port: u16, schema: &str) -> Result<()> { + let body = format!( + "[ch]\n\ + host = \"127.0.0.1\"\n\ + port = {ch_port}\n\ + database = \"default\"\n\ + compression = \"lz4\"\n\ + \n\ + [toast]\n\ + mode = \"clickhouse\"\n\ + \n\ + [table.\"{schema}\".\"t\"]\n\ + target_database = \"default\"\n\ + target_table = \"t\"\n\ + columns = [\n \ + {{ attnum = 1, target = \"id\", type = \"Int32\" }},\n \ + {{ attnum = 2, target = \"body\", type = \"String\" }},\n\ + ]\n", + ); + fs::write(path, body).with_context(|| format!("write ch-config {}", path.display()))?; + Ok(()) +} + +fn create_ch_dest_table(ch: &fx::ChServer) -> Result<()> { + ch.query("CREATE DATABASE IF NOT EXISTS default")?; + ch.query( + "CREATE OR REPLACE TABLE default.t (\ + id Int32,\ + body String,\ + _lsn UInt64,\ + _xid UInt32,\ + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool\ + ) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + )?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn dead_and_aborted_external_values_stay_out_of_ch() { + if !fx::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + + load_toast_workload(&source, "s19").expect("load toast workload"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + create_ch_dest_table(&ch).expect("create ch table"); + + let ch_config_path = tmp.path().join("ch-config.toml"); + write_toast_ch_config(&ch_config_path, slot.ch_tcp, "s19").expect("write ch-config"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn(&source, &ch_config_path, slot.walsender, &[]) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result = (|| -> Result<()> { + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(30)) + .context("daemon metrics endpoint never came up")?; + + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &N_LIVE.to_string(), + Duration::from_secs(60), + )?; + let ghosts = ch + .query("SELECT count() FROM default.t FINAL WHERE id >= 100") + .context("ghost count")?; + ensure!(ghosts == "0", "dead or aborted rows reached CH: {ghosts}"); + + // Compare body bytes to catch wrong generation or short reassembly + for id in 1..=N_LIVE { + let want: String = source + .psql_one(&format!("SELECT md5(body) FROM s19.t WHERE id = {id}")) + .with_context(|| format!("source body digest id={id}"))? + .trim() + .into(); + let got = ch + .query(&format!( + "SELECT lower(hex(MD5(body))) FROM default.t FINAL WHERE id = {id}" + )) + .with_context(|| format!("ch body digest id={id}"))?; + ensure!(got == want, "body id={id} differs: ch={got} source={want}"); + } + + // Require latest body version + let fresh = ch + .query("SELECT startsWith(body, 'fresh-1---') FROM default.t FINAL WHERE id = 1") + .context("updated body probe")?; + ensure!(fresh == "1", "id=1 kept the superseded body"); + + // Repair tally proves COPY path ran + let stderr = daemon.stderr(); + let line = stderr + .lines() + .find(|l| l.contains("visibility repair read pending relations")) + .context("repair never logged its read")?; + ensure!(line.contains("relations=1"), "repair read nothing: {line}"); + ensure!( + line.contains(&format!("rows={N_LIVE}")), + "repair row count off: {line}" + ); + + // The rows came from COPY, the chunks still went to the mirror + let toast_relid = source + .psql_one("SELECT reltoastrelid FROM pg_class WHERE oid = 's19.t'::regclass") + .context("source toast relid")?; + let mirror = format!("pg_toast_{toast_relid}"); + let created = ch + .query(&format!( + "SELECT count() FROM system.tables \ + WHERE database = 'default' AND name = '{mirror}'" + )) + .context("chunk mirror table probe")?; + ensure!(created == "1", "walk seeded no chunk mirror {mirror}"); + let mirrored = ch + .query(&format!("SELECT count() FROM default.`{mirror}`")) + .context("chunk mirror row probe")?; + ensure!( + mirrored.parse::().unwrap_or(0) > 0, + "chunk mirror {mirror} is empty" + ); + + // Non-TOAST update forces pointer resolution from seeded mirror + let before = ch + .query("SELECT max(_lsn) FROM default.t") + .context("pre-update lsn")?; + source + .psql_one("UPDATE s19.t SET id = id WHERE id = 2") + .context("post-bootstrap update")?; + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let now = ch + .query("SELECT max(_lsn) FROM default.t") + .context("post-update lsn")?; + if now != before { + break; + } + if Instant::now() >= deadline { + bail!("post-bootstrap update never reached CH: still at _lsn {before}"); + } + std::thread::sleep(Duration::from_millis(200)); + } + let want: String = source + .psql_one("SELECT md5(body) FROM s19.t WHERE id = 2") + .context("source body digest after update")? + .trim() + .into(); + let got = ch + .query("SELECT lower(hex(MD5(body))) FROM default.t FINAL WHERE id = 2") + .context("ch body digest after update")?; + ensure!( + got == want, + "unchanged pointer lost its body: ch={got} source={want}" + ); + Ok(()) + })(); + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_types_e2e.rs b/tests/bootstrap_types_e2e.rs index 14f12502..36f55cc8 100644 --- a/tests/bootstrap_types_e2e.rs +++ b/tests/bootstrap_types_e2e.rs @@ -30,19 +30,6 @@ fn extension_available(name: &str) -> bool { .exists() } -fn make_source(tmp: &tempfile::TempDir) -> Shadow { - let mut cfg = ShadowConfig::new( - tmp.path().join("source-data"), - tmp.path().join("source-filtered"), - ); - cfg.port = fx::PG_SOURCE_PORT; - cfg.socket_dir = tmp.path().join("source-sock"); - cfg.ctl_timeout = Duration::from_secs(60); - fs::create_dir_all(&cfg.filter_out_dir).unwrap(); - fs::create_dir_all(&cfg.socket_dir).unwrap(); - Shadow::new(cfg) -} - fn load_types_workload(source: &Shadow, has_postgis: bool) -> Result<()> { let mut cols = String::from( "id int PRIMARY KEY, c_bool bool, c_int2 smallint, c_int4 int, c_int8 bigint, \ @@ -105,8 +92,7 @@ fn write_autocreate_config(path: &Path, ch_port: u16) -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn direct_bootstrap_all_types_end_to_end() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } for ext in ["hstore", "citext", "vector"] { @@ -120,7 +106,7 @@ async fn direct_bootstrap_all_types_end_to_end() { let slot = fx::Ports::alloc(); let tmp = tempfile::tempdir().unwrap(); - let source = make_source(&tmp); + let source = fx::make_source(&tmp); source.initdb().expect("initdb source"); source.write_base_conf().expect("source base conf"); fx::append_source_conf(&source).expect("append source conf"); diff --git a/tests/bootstrap_window_ch.rs b/tests/bootstrap_window_ch.rs new file mode 100644 index 00000000..06531fc1 --- /dev/null +++ b/tests/bootstrap_window_ch.rs @@ -0,0 +1,187 @@ +//! Verify window leg ships commits below pump resume boundary + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::io::Write as _; +use std::time::Duration; + +use anyhow::{Context, Result}; +use walshadow::mapping::TableTarget; +use walshadow::schema::RelName; + +const N_ROWS: i32 = 64; +/// Keep backup window open for writer +const MAX_RATE_KIB: &str = "32768"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn window_writes_reach_ch() { + if !fx::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + + fx::load_source_workload(&source, "s15", N_ROWS).expect("load source workload"); + source + .apply_schema_dump( + "CREATE TABLE s15.fixed (id int PRIMARY KEY, n bigint); + ALTER TABLE s15.fixed REPLICA IDENTITY FULL; + INSERT INTO s15.fixed SELECT g, g FROM generate_series(1, 64) g; + CHECKPOINT;", + ) + .unwrap(); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + fx::create_ch_dest_table(&ch, "default", "t").expect("create ch table"); + ch.query( + "CREATE TABLE default.fixed (id Int32, n Int64, _lsn UInt64, _xid UInt32, + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool) + ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + ) + .unwrap(); + + let ch_config_path = tmp.path().join("ch-config.toml"); + fx::write_ch_config_toml( + &ch_config_path, + "127.0.0.1", + slot.ch_tcp, + "default", + &RelName::new("s15", "t"), + &TableTarget::new("default", "t"), + ) + .expect("write ch-config"); + let mut config = std::fs::OpenOptions::new() + .append(true) + .open(&ch_config_path) + .unwrap(); + writeln!( + config, + "\n[table.\"s15\".\"fixed\"] + target_database = \"default\" + target_table = \"fixed\" + columns = [ + {{ attnum = 1, target = \"id\", type = \"Int32\" }}, + {{ attnum = 2, target = \"n\", type = \"Int64\" }}, + ]" + ) + .unwrap(); + drop(config); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn( + &source, + &ch_config_path, + slot.walsender, + // First-tick slot retains window segments + &[ + "--bootstrap-max-rate-kib", + MAX_RATE_KIB, + "--slot", + "walshadow_window", + ], + ) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result = (|| -> Result<()> { + fx::wait_for_backup_streaming(&source, Duration::from_secs(60))?; + + // Put batch below pump resume segment + source + .apply_schema_dump( + "INSERT INTO s15.t SELECT g, 'window-'||g::text \ + FROM generate_series(1001, 1100) g;\n\ + SELECT pg_switch_wal();\n\ + UPDATE s15.t SET name = 'updated-'||id::text WHERE id <= 32;\n\ + INSERT INTO s15.fixed SELECT g, -g FROM generate_series(1001, 1100) g;\n\ + UPDATE s15.fixed SET n = -id WHERE id <= 32;\n\ + DELETE FROM s15.fixed WHERE id > 32 AND id < 1000;\n\ + SELECT pg_switch_wal();\n", + ) + .context("window batch below the pump's resume")?; + + // Keep source advancing until backup closes + let mut round = 0; + while fx::backup_in_progress(&source) { + round += 1; + source + .apply_schema_dump(&format!( + "INSERT INTO s15.t SELECT g, 'late-'||g::text \ + FROM generate_series({from}, {to}) g;\n\ + DELETE FROM s15.t WHERE id = {del};\n", + from = 2000 + round * 10, + to = 2009 + round * 10, + del = 33 + round, + )) + .context("in-window writes")?; + std::thread::sleep(Duration::from_millis(50)); + } + + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(60)) + .context("daemon metrics endpoint never came up")?; + + let src_count = source + .psql_one("SELECT count(*) FROM s15.t") + .context("source count")?; + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE _is_deleted = 0", + &src_count, + Duration::from_secs(60), + ) + .context("window rows never reached CH")?; + fx::assert_ch_matches_source(&ch, &source, "s15.t", "default.t") + .context("source vs CH parity across the backup window")?; + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.fixed FINAL WHERE _is_deleted = 0", + "132", + Duration::from_secs(60), + )?; + let expected = source.psql_one( + "SELECT string_agg(id::text || ':' || n::text, ',' ORDER BY id) FROM s15.fixed", + )?; + let actual = ch.query( + "SELECT arrayStringConcat(groupArray(concat(toString(id), ':', toString(n))), ',') + FROM (SELECT id, n FROM default.fixed FINAL WHERE _is_deleted = 0 ORDER BY id)", + )?; + anyhow::ensure!(actual == expected, "fixed-width rows differ: {actual}"); + anyhow::ensure!( + ch.query( + "SELECT count() FROM default.fixed FINAL WHERE _is_deleted = 0 AND _commit_ts = 0" + )? == "0", + "fixed-width changes must carry WAL commit timestamps" + ); + + // Leg tally distinguishes delivery from pump overlap + let log = daemon.stderr(); + let shipped = log + .lines() + .find(|l| l.contains("backup window shipped")) + .context("daemon logged no window-leg summary")?; + anyhow::ensure!( + !shipped.contains("rows=0"), + "window leg shipped no rows: {shipped}", + ); + let gate = log + .lines() + .find(|l| l.contains("bootstrap visibility gate settled")) + .context("no gate summary")?; + anyhow::ensure!( + gate.contains("pending_relations=0"), + "unexpected source repair: {gate}" + ); + Ok(()) + })(); + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_window_leg_ch.rs b/tests/bootstrap_window_leg_ch.rs new file mode 100644 index 00000000..0350c855 --- /dev/null +++ b/tests/bootstrap_window_leg_ch.rs @@ -0,0 +1,287 @@ +//! Verify live window leg reads through exact `end_lsn` and patches commits + +#![cfg(target_os = "linux")] + +#[path = "common/inproc_harness.rs"] +mod fx; + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use walrus::pg::backup::{format_pg_lsn, parse_pg_lsn}; +use walshadow::backfill_bootstrap::seed_catalog_from_source; +use walshadow::bootstrap_window::{ + WindowLegConfig, replay_segments, segments_in_dir, stream_window, +}; +use walshadow::ch::CompressionChoice; +use walshadow::ch_emitter::{EmitterConfig, EmitterStats}; +use walshadow::config::ResolvedConfig; +use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; +use walshadow::pipeline::Fatal; +use walshadow::record::WAL_SEG_SIZE; +use walshadow::schema::RelName; +use walshadow::source_feed::{SourceFeed, open_sql_client}; +use walshadow::toast::{MemChunkStore, ToastResolver}; +use walshadow::visibility::PgXactPatch; +use walshadow::wal_stream::WalStream; + +const SCHEMA: &str = "s23"; +const N_ROWS: i32 = 100; + +fn emitter(port: u16) -> EmitterConfig { + let mut cfg = EmitterConfig { + host: "127.0.0.1".into(), + port, + database: "walshadow_test".into(), + compression: CompressionChoice::None, + flush_timeout: Duration::from_millis(50), + ..Default::default() + }; + cfg.tables.insert( + RelName::new(SCHEMA, "t"), + TableMapping { + target: TableTarget::new("walshadow_test", "t"), + columns: vec![ + ColumnMapping { + src_attnum: 1, + target_name: "id".into(), + target_type: "Int32".into(), + }, + ColumnMapping { + src_attnum: 2, + target_name: "name".into(), + target_type: "String".into(), + }, + ], + }, + ); + cfg +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn partial_transaction_failure_returns_from_live_and_archived_replay() { + if !fx::requirements_available() { + return; + } + let ports = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + let source = fx::make_pg(&tmp, "source", ports.source); + source.initdb().unwrap(); + source.write_base_conf().unwrap(); + fx::append_source_conf(&source); + source.start().unwrap(); + let _stop = fx::StopOnDrop { sh: &source }; + source + .apply_schema_dump(&format!( + "CREATE SCHEMA {SCHEMA}; + CREATE TABLE {SCHEMA}.t (id int PRIMARY KEY, name text); + ALTER TABLE {SCHEMA}.t ALTER COLUMN name SET STORAGE EXTERNAL; + ALTER TABLE {SCHEMA}.t REPLICA IDENTITY FULL; + INSERT INTO {SCHEMA}.t VALUES (2, repeat('external', 1000)); + SELECT pg_switch_wal();" + )) + .unwrap(); + let ch = + fx::ChServer::spawn(tempfile::tempdir().unwrap(), ports.ch_tcp, ports.ch_http).unwrap(); + ch.query("CREATE DATABASE walshadow_test").unwrap(); + ch.query( + "CREATE TABLE walshadow_test.t (id Int32, name String, _lsn UInt64, _xid UInt32, + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool) + ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + ) + .unwrap(); + let pg = fx::pg_cfg(&source, "window-error-test"); + let sql = open_sql_client(&pg).await.unwrap(); + let catalog = seed_catalog_from_source(&sql).await.unwrap(); + let mut feed = SourceFeed::connect(&pg).await.unwrap(); + let ident = feed.identify_system().await.unwrap(); + sql.batch_execute(&format!( + "BEGIN; INSERT INTO {SCHEMA}.t VALUES (1, 'inline'); + UPDATE {SCHEMA}.t SET id=id WHERE id=2; COMMIT" + )) + .await + .unwrap(); + let end = walshadow::pg::current_wal_lsn(&sql).await.unwrap(); + sql.batch_execute("SELECT pg_switch_wal()").await.unwrap(); + let stats = Arc::new(EmitterStats::default()); + let emitter = emitter(ports.ch_tcp); + let cfg = WindowLegConfig { + mapping: walshadow::mapping::mapping_handle(emitter.tables.clone()), + emitter, + config: Arc::new(ResolvedConfig::default()), + stats: stats.clone(), + resolver: ToastResolver::with_store(Arc::new(MemChunkStore::new()), stats), + oracle: None, + fatal: Fatal::new(), + scratch_dir: tmp.path().join("leg"), + patch: Arc::new(std::sync::Mutex::new(PgXactPatch::new())), + catalog, + pg_major: (feed.server_version_num() / 10000) as u32, + system_id: ident.sysid, + timeline: ident.timeline, + }; + let (_stop_tx, stop_rx) = tokio::sync::watch::channel(Some(end)); + let err = tokio::time::timeout( + Duration::from_secs(10), + stream_window(cfg.clone(), &mut feed, ident.xlogpos, stop_rx), + ) + .await + .expect("live replay hung after partial transaction") + .unwrap_err(); + assert!(format!("{err:#}").contains("no mirror"), "{err:#}"); + + let segments = segments_in_dir( + &source.config().data_dir.join("pg_wal"), + ident.timeline, + ident.xlogpos, + end, + ) + .await + .unwrap(); + let err = tokio::time::timeout( + Duration::from_secs(10), + replay_segments(cfg, &segments, ident.xlogpos, end), + ) + .await + .expect("archived replay hung after partial transaction") + .unwrap_err(); + assert!(format!("{err:#}").contains("no mirror"), "{err:#}"); + assert_eq!( + ch.query("SELECT id FROM walshadow_test.t FINAL").unwrap(), + "1" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn leg_reads_through_end_lsn_inside_its_first_segment() { + if !fx::pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + if !fx::clickhouse_available() { + eprintln!("skip: no clickhouse binary on PATH"); + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::make_pg(&tmp, "source", slot.source); + source.initdb().expect("initdb source"); + source.write_base_conf().expect("source base conf"); + fx::append_source_conf(&source); + source.start().expect("start source"); + let _src_stop = fx::StopOnDrop { sh: &source }; + // Fresh segment, so the window below lives in one segment with room + source + .apply_schema_dump(&format!( + "CREATE SCHEMA {SCHEMA};\n\ + CREATE TABLE {SCHEMA}.t (id int4 PRIMARY KEY, name text NOT NULL);\n\ + ALTER TABLE {SCHEMA}.t REPLICA IDENTITY FULL;\n\ + SELECT pg_switch_wal();\n" + )) + .expect("source schema"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + ch.query("CREATE DATABASE IF NOT EXISTS walshadow_test") + .expect("create db"); + ch.query( + "CREATE OR REPLACE TABLE walshadow_test.t (\ + id Int32,\ + name String,\ + _lsn UInt64,\ + _xid UInt32,\ + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool\ + ) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + ) + .expect("create dest table"); + + let result: Result<()> = async { + let pg = fx::pg_cfg(&source, "window-leg-test"); + let sql = open_sql_client(&pg).await.context("sidecar connect")?; + let catalog = seed_catalog_from_source(&sql) + .await + .context("seed catalog")?; + let mut feed = SourceFeed::connect(&pg).await.context("feed connect")?; + let ident = feed.identify_system().await.context("IDENTIFY_SYSTEM")?; + let from_lsn = ident.xlogpos; + + sql.batch_execute(&format!( + "INSERT INTO {SCHEMA}.t SELECT g, 'leg-'||g::text FROM generate_series(1, {N_ROWS}) g" + )) + .await + .context("window INSERT")?; + let end: String = sql + .query_one("SELECT pg_current_wal_flush_lsn()::text", &[]) + .await + .context("flush lsn")? + .get(0); + let end_lsn = parse_pg_lsn(&end)?; + ensure!( + WalStream::align_down(from_lsn, WAL_SEG_SIZE) + == WalStream::align_down(end_lsn, WAL_SEG_SIZE), + "window crossed a segment: from={} end={end}", + format_pg_lsn(from_lsn), + ); + // One record past `end_lsn`, so the leg's position can pass it + sql.batch_execute("SELECT txid_current()") + .await + .context("trailing commit")?; + + let (stop_tx, stop_rx) = tokio::sync::watch::channel(Some(end_lsn)); + let patch = Arc::new(std::sync::Mutex::new(PgXactPatch::new())); + let cfg = WindowLegConfig { + emitter: emitter(slot.ch_tcp), + mapping: walshadow::mapping::mapping_handle(emitter(slot.ch_tcp).tables), + config: Arc::new(ResolvedConfig::default()), + stats: Arc::new(EmitterStats::default()), + resolver: ToastResolver::disabled(), + oracle: None, + fatal: Fatal::new(), + scratch_dir: tmp.path().join("leg-scratch"), + patch: patch.clone(), + catalog, + pg_major: (feed.server_version_num() / 10000) as u32, + system_id: ident.sysid.clone(), + timeline: ident.timeline, + }; + let leg = tokio::time::timeout( + Duration::from_secs(30), + stream_window(cfg, &mut feed, from_lsn, stop_rx), + ) + .await + .context("leg never covered end_lsn")? + .context("window leg")?; + drop(stop_tx); + + ensure!( + leg.through_lsn >= end_lsn, + "leg sealed at {} below end_lsn {end}", + format_pg_lsn(leg.through_lsn), + ); + ensure!( + leg.replay.rows_replayed >= N_ROWS as u64, + "leg shipped {} rows, want {N_ROWS}", + leg.replay.rows_replayed, + ); + let patched = patch.lock().unwrap().len(); + ensure!(patched >= 1, "leg patched no commit"); + let n = ch + .query(&format!( + "SELECT count() FROM walshadow_test.t FINAL WHERE id <= {N_ROWS}" + )) + .context("ch count")?; + ensure!( + n == N_ROWS.to_string(), + "CH holds {n} of {N_ROWS} window rows" + ); + Ok(()) + } + .await; + + if let Err(e) = result { + panic!("{e:#}"); + } +} diff --git a/tests/common/bootstrap_ch_fixture.rs b/tests/common/bootstrap_ch_fixture.rs index 2a9f7179..298b8149 100644 --- a/tests/common/bootstrap_ch_fixture.rs +++ b/tests/common/bootstrap_ch_fixture.rs @@ -1,13 +1,12 @@ -//! Shared scaffolding for the bootstrap → CH end-to-end -//! drills (`bootstrap_direct_ch.rs`, -//! `bootstrap_object_store_ch.rs`). +//! Shared scaffolding for the `bootstrap_*_ch.rs` end-to-end drills. //! //! Owns: the `ChServer` subprocess wrapper (lifted from -//! `pipeline_e2e.rs` so the pipeline DDL drill, both bootstrap drills, +//! `pipeline_e2e.rs` so the pipeline DDL drill, the bootstrap drills, //! and the kill-restart drill share one driver), TOML CH-config //! rendering for the table mapping the daemon consumes via -//! `--ch-config`, and the `assert_ch_matches_source` count/sum/md5 -//! oracle the two drills share. +//! `--ch-config`, the `assert_ch_matches_source` count/sum/md5 oracle, +//! and the source / `DaemonRun` / teardown scaffolding each drill +//! repeats verbatim. //! //! Included from the test files via `#[path = "common/..."]` rather //! than wired through `tests/common/mod.rs` because Cargo would @@ -18,7 +17,7 @@ #[path = "ports.rs"] mod ports; #[allow(unused_imports)] -pub use ports::{PG_SHADOW_PORT, PG_SOURCE_PORT, Ports, reserve_port, reserve_span}; +pub use ports::{PG_SHADOW_PORT, PG_SOURCE_PORT, Ports, pg_cfg, reserve_port, reserve_span}; use std::fs; use std::io::Write; @@ -31,7 +30,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use walshadow::mapping::TableTarget; use walshadow::schema::RelName; -use walshadow::shadow::{BridgeConf, Shadow}; +use walshadow::shadow::{BridgeConf, Shadow, ShadowConfig}; /// ClickHouse server subprocess wrapper shared by the pipeline DDL /// drill, both bootstrap-to-CH drills, and the @@ -153,6 +152,178 @@ impl Drop for ChServer { } } +/// Source cluster under `tmp`, socket-only on the source port. +pub fn make_source(tmp: &tempfile::TempDir) -> Shadow { + let mut cfg = ShadowConfig::new( + tmp.path().join("source-data"), + tmp.path().join("source-filtered"), + ); + cfg.port = ports::PG_SOURCE_PORT; + cfg.socket_dir = tmp.path().join("source-sock"); + cfg.ctl_timeout = Duration::from_secs(60); + fs::create_dir_all(&cfg.filter_out_dir).unwrap(); + fs::create_dir_all(&cfg.socket_dir).unwrap(); + Shadow::new(cfg) +} + +/// Whether PostgreSQL reports a base backup in flight. +pub fn backup_in_progress(source: &Shadow) -> bool { + let n = source + .psql_one("SELECT count(*) FROM pg_stat_progress_basebackup") + .unwrap_or_default(); + n.trim() != "0" && !n.trim().is_empty() +} + +/// Wait until the daemon's `BASE_BACKUP` shows up on the source, the cue +/// for a drill to write inside the backup window. +pub fn wait_for_backup_streaming(source: &Shadow, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + while !backup_in_progress(source) { + if Instant::now() >= deadline { + bail!("BASE_BACKUP never showed up in pg_stat_progress_basebackup"); + } + std::thread::sleep(Duration::from_millis(20)); + } + Ok(()) +} + +/// Daemon-side layout of one bootstrap-to-CH drill: the dirs the flags +/// name, plus where the daemon's stderr lands. +pub struct DaemonRun { + pub shadow_data_dir: PathBuf, + pub shadow_sock: PathBuf, + pub filter_dir: PathBuf, + pub spill_dir: PathBuf, + pub stderr_path: PathBuf, + pub metrics_addr: std::net::SocketAddr, +} + +impl DaemonRun { + /// Create every dir the daemon expects to exist. The shadow data dir + /// stays absent: bootstrap refuses to land on a populated one. + pub fn prepare(tmp: &Path, metrics_port: u16) -> Result { + let run = Self { + shadow_data_dir: tmp.join("shadow-data"), + shadow_sock: tmp.join("shadow-sock"), + filter_dir: tmp.join("filtered"), + spill_dir: tmp.join("spill"), + stderr_path: tmp.join("daemon.stderr.log"), + metrics_addr: format!("127.0.0.1:{metrics_port}").parse()?, + }; + for d in [&run.shadow_sock, &run.filter_dir, &run.spill_dir] { + fs::create_dir_all(d).with_context(|| format!("create {}", d.display()))?; + } + Ok(run) + } + + /// Spawn `walshadow-stream` in `--bootstrap-mode direct` against + /// `source`, appending `extra` flags. + pub fn spawn( + &self, + source: &Shadow, + ch_config: &Path, + walsender_port: u16, + extra: &[&str], + ) -> Result { + self.spawn_mode(source, ch_config, walsender_port, "direct", extra, &[]) + } + + /// [`Self::spawn`] in another bootstrap mode, with the command + /// environment that mode reads (object store resolves libpq vars). Its + /// own process group, so a shadow it spawned dies with the kill in + /// `ChildGuard`. + pub fn spawn_mode( + &self, + source: &Shadow, + ch_config: &Path, + walsender_port: u16, + mode: &str, + extra: &[&str], + env: &[(&str, String)], + ) -> Result { + let stderr = fs::File::create(&self.stderr_path) + .with_context(|| format!("create {}", self.stderr_path.display()))?; + Command::new(env!("CARGO_BIN_EXE_walshadow-stream")) + .args([ + "--host", + source.config().socket_dir.to_str().unwrap(), + "--port", + &source.config().port.to_string(), + "--user", + "postgres", + "--dbname", + "postgres", + "--sslmode", + "disable", + "--out-dir", + self.filter_dir.to_str().unwrap(), + "--shadow-socket-dir", + self.shadow_sock.to_str().unwrap(), + "--shadow-port", + &ports::PG_SHADOW_PORT.to_string(), + "--shadow-user", + "postgres", + "--shadow-dbname", + "postgres", + "--spill-dir", + self.spill_dir.to_str().unwrap(), + "--status-interval", + "1", + "--metrics-bind", + &self.metrics_addr.to_string(), + "--walsender-bind", + &format!("127.0.0.1:{walsender_port}"), + "--retention-bytes", + "0", + "--ch-config", + ch_config.to_str().unwrap(), + "--bootstrap-mode", + mode, + "--bootstrap-shadow-data-dir", + self.shadow_data_dir.to_str().unwrap(), + "--bootstrap-shadow-replay-timeout", + "120", + ]) + .args(extra) + .envs(env.iter().map(|(k, v)| (*k, v))) + .env("RUST_LOG", "warn,walshadow=info") + .stdout(Stdio::null()) + .stderr(Stdio::from(stderr)) + .process_group(0) + .spawn() + .context("spawn walshadow-stream") + } + + /// Everything the daemon has logged so far. + pub fn stderr(&self) -> String { + fs::read_to_string(&self.stderr_path).unwrap_or_default() + } + + /// Wait for a needle to appear in the daemon's log. + pub fn wait_for_log(&self, needle: &str, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + while !self.stderr().contains(needle) { + if Instant::now() >= deadline { + bail!("daemon never logged {needle:?}"); + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(()) + } + + /// Stop a shadow the daemon left running after its own exit. + pub fn stop_shadow(&self) { + if !self.shadow_data_dir.join("postmaster.pid").exists() { + return; + } + let mut cfg = ShadowConfig::new(self.shadow_data_dir.clone(), self.filter_dir.clone()); + cfg.port = ports::PG_SHADOW_PORT; + cfg.socket_dir = self.shadow_sock.clone(); + cfg.ctl_timeout = Duration::from_secs(60); + let _ = Shadow::new(cfg).stop(); + } +} + /// Skip-gate probe — same shape as `pipeline_e2e.rs::clickhouse_available`. pub fn clickhouse_available() -> bool { Command::new("clickhouse") @@ -184,6 +355,62 @@ pub fn pg_basebackup_available() -> bool { .unwrap_or(false) } +/// Skip gate every daemon bootstrap drill shares. Names the missing tool. +pub fn requirements_available() -> bool { + for (tool, found) in [ + ("initdb", pg_available()), + ("pg_basebackup", pg_basebackup_available()), + ("clickhouse", clickhouse_available()), + ] { + if !found { + eprintln!("skip: no {tool} on PATH"); + return false; + } + } + true +} + +/// Source cluster under `tmp`, initialised with the bootstrap overrides and +/// running. Caller keeps it alive and wraps it in [`StopOnDrop`]. +pub fn start_source(tmp: &tempfile::TempDir) -> Shadow { + let source = make_source(tmp); + source.initdb().expect("initdb source"); + source.write_base_conf().expect("source base conf"); + append_source_conf(&source).expect("append source conf"); + source.start().expect("start source"); + source +} + +/// Poll `sql` on CH until it answers `want`. The tail drains asynchronously, +/// so racing it with an immediate assert flakes on slow CI. +pub fn wait_for_ch_value(ch: &ChServer, sql: &str, want: &str, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + let got = ch.query(sql).unwrap_or_default(); + if got == want { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("CH never answered {want:?} to `{sql}`, last {got:?}"); + } + std::thread::sleep(Duration::from_millis(200)); + } +} + +/// Kill the daemon before the shadow, so its supervisor cannot restart the +/// postmaster, then stop whatever SIGKILL left behind and report `result` +/// with the daemon's log attached. +pub fn finish_daemon(guard: ChildGuard, daemon: &DaemonRun, result: Result<()>) { + if let Some(mut child) = guard.into_inner() { + let _ = child.kill(); + let _ = child.wait(); + } + daemon.stop_shadow(); + if let Err(e) = result { + panic!("{e:#}\n--- daemon stderr ---\n{}", daemon.stderr()); + } +} + /// Build tree holding `walshadow.so`, fed to PG as `dynamic_library_path`. /// Daemon dials the bridge worker at boot, so an unbuilt module is a failure, /// not a reason to skip diff --git a/tests/common/inproc_harness.rs b/tests/common/inproc_harness.rs index 5721f683..f5b790f5 100644 --- a/tests/common/inproc_harness.rs +++ b/tests/common/inproc_harness.rs @@ -20,7 +20,7 @@ #[path = "ports.rs"] mod ports; #[allow(unused_imports)] -pub use ports::{PG_SHADOW_PORT, PG_SOURCE_PORT, Ports, reserve_port, reserve_span}; +pub use ports::{PG_SHADOW_PORT, PG_SOURCE_PORT, Ports, pg_cfg, reserve_port, reserve_span}; use std::fs; use std::io::Write as _; @@ -33,8 +33,6 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use tokio::sync::Mutex; -use walrus::pg::replication::conn::PgConfig; -use walrus::pg::replication::tls::{SslMode, TlsParams}; use walshadow::ch::CompressionChoice; use walshadow::ch_ddl::{DdlApplicator, DdlConfig}; @@ -89,6 +87,20 @@ pub fn clickhouse_available() -> bool { .unwrap_or(false) } +pub fn requirements_available() -> bool { + for (tool, found) in [ + ("initdb", pg_available()), + ("pg_basebackup", pg_basebackup_available()), + ("clickhouse", clickhouse_available()), + ] { + if !found { + eprintln!("skip: no {tool} on PATH"); + return false; + } + } + true +} + // --------------------------------------------------------------------------- // PG fixture helpers // --------------------------------------------------------------------------- @@ -675,17 +687,7 @@ async fn build_pipeline_inner( app_name, ddl, } = args; - let scfg = source.config(); - let pgcfg = PgConfig { - host: scfg.socket_dir.to_string_lossy().into_owned(), - port: scfg.port, - user: "postgres".into(), - password: None, - database: "postgres".into(), - application_name: app_name.into(), - sslmode: SslMode::Disable, - tls: TlsParams::default(), - }; + let pgcfg = pg_cfg(source, app_name); let mut feed = SourceFeed::connect(&pgcfg) .await .expect("source feed connect") diff --git a/tests/common/ports.rs b/tests/common/ports.rs index 561e0211..8057fb32 100644 --- a/tests/common/ports.rs +++ b/tests/common/ports.rs @@ -12,6 +12,10 @@ //! sibling test probing that port meanwhile would find it free. Lock files //! sit under `TMPDIR`, so concurrent test processes must share one. //! +//! Also hands out the socket [`PgConfig`] a test dials a fixture cluster on, +//! the other half of naming one: `application_name` says which drill holds a +//! connection when `pg_stat_activity` is read. +//! //! Postgres clusters here are socket-only (`listen_addresses = ''`) and PG keys //! its SysV segment off the data dir inode (PG `src/backend/port/sysv_shmem.c`), //! so a cluster's `port` only names a socket file inside a per-test temp dir. @@ -25,6 +29,9 @@ use std::path::PathBuf; use std::sync::Mutex; use ahash::{HashSet, HashSetExt}; +use walrus::pg::replication::conn::PgConfig; +use walrus::pg::replication::tls::{SslMode, TlsParams}; +use walshadow::shadow::Shadow; /// Socket-only cluster ports. Distinct per role so a test that puts source and /// shadow in one socket dir still gets distinct socket files. @@ -131,3 +138,17 @@ impl Ports { } } } + +/// Socket connection parameters for a fixture cluster +pub fn pg_cfg(sh: &Shadow, application_name: &str) -> PgConfig { + PgConfig { + host: sh.config().socket_dir.to_str().unwrap().to_string(), + port: sh.config().port, + user: "postgres".into(), + password: None, + database: "postgres".into(), + application_name: application_name.into(), + sslmode: SslMode::Disable, + tls: TlsParams::default(), + } +} diff --git a/tests/composite_pkey.rs b/tests/composite_pkey.rs index e60084cd..e777ae42 100644 --- a/tests/composite_pkey.rs +++ b/tests/composite_pkey.rs @@ -123,8 +123,7 @@ fn live_pairs(ch: &fx::ChServer) -> String { } fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/control_plane_e2e.rs b/tests/control_plane_e2e.rs index d35daaa9..3f56d142 100644 --- a/tests/control_plane_e2e.rs +++ b/tests/control_plane_e2e.rs @@ -635,19 +635,7 @@ fn spawn_daemon(bin: &str, args: &[String], stderr_path: &Path) -> Result } fn gated() -> bool { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return false; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return false; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); - return false; - } - true + fx::requirements_available() } const USER_EMAIL: &str = diff --git a/tests/copy_into.rs b/tests/copy_into.rs index b095c64a..0de9ac04 100644 --- a/tests/copy_into.rs +++ b/tests/copy_into.rs @@ -28,16 +28,7 @@ const N_ROWS: u32 = 500; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn copy_into_multi_insert_replicates() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } diff --git a/tests/ddl_replicates.rs b/tests/ddl_replicates.rs index aa10772a..dbeaea57 100644 --- a/tests/ddl_replicates.rs +++ b/tests/ddl_replicates.rs @@ -39,16 +39,7 @@ use walshadow::schema::RelName; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn alter_add_column_replicates_without_toml_edit() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } @@ -186,8 +177,7 @@ async fn alter_add_column_replicates_without_toml_edit() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn create_table_auto_replicates_in_namespace() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -286,8 +276,7 @@ async fn create_table_auto_replicates_in_namespace() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn drop_table_strategy_drop_removes_dest() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -385,8 +374,7 @@ async fn drop_table_strategy_drop_removes_dest() { /// missing table. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn pinned_mapping_create_drop_create_recreates_dest() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -521,8 +509,7 @@ async fn pinned_mapping_create_drop_create_recreates_dest() { /// the override and everything landed in the global DB. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn auto_create_honors_per_namespace_target_database() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -625,8 +612,7 @@ async fn auto_create_honors_per_namespace_target_database() { /// `EmitterConfig::from_toml_str` → resolve → `CREATE TABLE` path end-to-end. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn create_table_auto_replicates_from_toml_namespace() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/desc_log_e2e.rs b/tests/desc_log_e2e.rs index 8d86707c..d1b7232a 100644 --- a/tests/desc_log_e2e.rs +++ b/tests/desc_log_e2e.rs @@ -23,8 +23,7 @@ use walshadow::schema::RelName; /// dropping the new column's values. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn prepared_ddl_drains_at_commit_prepared() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return; } let slot = fx::Ports::alloc(); @@ -131,8 +130,7 @@ async fn prepared_ddl_drains_at_commit_prepared() { /// mapping keyed under it. With stale descriptors they skip as unmapped. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn schema_rename_reroutes_under_new_namespace() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return; } let slot = fx::Ports::alloc(); @@ -243,8 +241,7 @@ async fn in_place_intervals_compatible_and_ambiguous() { use walrus::pg::walparser::RelFileNode; use walshadow::desc_log::{AmbiguityReason, LookupResult}; - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return; } let slot = fx::Ports::alloc(); diff --git a/tests/desc_log_restart_e2e.rs b/tests/desc_log_restart_e2e.rs index 7d41d2cf..0699de45 100644 --- a/tests/desc_log_restart_e2e.rs +++ b/tests/desc_log_restart_e2e.rs @@ -126,8 +126,7 @@ fn mappings_for(namespace: &str, table: &str) -> Vec { /// records unseen — and must classify the commit from its inval set alone. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn rename_commit_after_restart_reroutes() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return; } let slot = fx::Ports::alloc(); @@ -224,8 +223,7 @@ async fn rename_commit_after_restart_reroutes() { /// xact's inval set; namespace catcache invals must force capture-all. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn prepared_rename_commit_after_restart_reroutes() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return; } let slot = fx::Ports::alloc(); diff --git a/tests/dirty_admission_e2e.rs b/tests/dirty_admission_e2e.rs index 32741622..6ed4a37b 100644 --- a/tests/dirty_admission_e2e.rs +++ b/tests/dirty_admission_e2e.rs @@ -21,8 +21,7 @@ use walshadow::mapping::NamespaceMapping; use walshadow::shadow::Shadow; fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return true; } false diff --git a/tests/foreign_database_e2e.rs b/tests/foreign_database_e2e.rs index bd997c56..4922bc07 100644 --- a/tests/foreign_database_e2e.rs +++ b/tests/foreign_database_e2e.rs @@ -16,8 +16,7 @@ use walshadow::mapping::NamespaceMapping; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn foreign_database_ddl_and_dml_never_reach_the_followed_output() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/init_e2e.rs b/tests/init_e2e.rs index 804d6770..4f3c0afd 100644 --- a/tests/init_e2e.rs +++ b/tests/init_e2e.rs @@ -17,26 +17,11 @@ mod fx; use std::fs; -use std::time::Duration; use walshadow::ch_emitter::EmitterConfig; use walshadow::config::SourceConn; use walshadow::init::{InitOpts, run}; use walshadow::schema::RelName; -use walshadow::shadow::{Shadow, ShadowConfig}; - -fn make_source(tmp: &tempfile::TempDir) -> Shadow { - let mut cfg = ShadowConfig::new( - tmp.path().join("source-data"), - tmp.path().join("source-filtered"), - ); - cfg.port = fx::PG_SOURCE_PORT; - cfg.socket_dir = tmp.path().join("source-sock"); - cfg.ctl_timeout = Duration::from_secs(60); - fs::create_dir_all(&cfg.filter_out_dir).unwrap(); - fs::create_dir_all(&cfg.socket_dir).unwrap(); - Shadow::new(cfg) -} /// Percent-encode a socket dir into the libpq URL spelling fn socket_url(socket_dir: &std::path::Path, dbname: &str) -> String { @@ -61,7 +46,7 @@ async fn init_probes_both_ends_and_writes_a_bootable_config() { let slot = fx::Ports::alloc(); let tmp = tempfile::tempdir().unwrap(); - let source = make_source(&tmp); + let source = fx::make_source(&tmp); source.initdb().expect("initdb source"); source.write_base_conf().expect("source base conf"); fx::append_source_conf(&source).expect("append source conf"); diff --git a/tests/kill_restart.rs b/tests/kill_restart.rs index ecb5cb03..9ef40488 100644 --- a/tests/kill_restart.rs +++ b/tests/kill_restart.rs @@ -389,16 +389,7 @@ async fn kill_restart_post_commit_preserves_end_state() { } async fn drill(strategy: Strategy) { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } diff --git a/tests/oracle_types_e2e.rs b/tests/oracle_types_e2e.rs index bf5cde9e..1a4ec7f6 100644 --- a/tests/oracle_types_e2e.rs +++ b/tests/oracle_types_e2e.rs @@ -20,8 +20,7 @@ use walshadow::schema::RelName; use walshadow::shadow::Shadow; fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/pending_capture_e2e.rs b/tests/pending_capture_e2e.rs index 2c884847..c8ed0cd1 100644 --- a/tests/pending_capture_e2e.rs +++ b/tests/pending_capture_e2e.rs @@ -34,8 +34,7 @@ use walshadow::mapping::NamespaceMapping; use walshadow::shadow::Shadow; fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse on PATH"); + if !fx::requirements_available() { return true; } false diff --git a/tests/pgbench_acceptance.rs b/tests/pgbench_acceptance.rs index 9c956acd..30d9db93 100644 --- a/tests/pgbench_acceptance.rs +++ b/tests/pgbench_acceptance.rs @@ -46,19 +46,6 @@ fn pgbench_available() -> bool { .unwrap_or(false) } -fn make_source(tmp: &tempfile::TempDir, port: u16) -> Shadow { - let mut cfg = ShadowConfig::new( - tmp.path().join("source-data"), - tmp.path().join("source-filtered"), - ); - cfg.port = port; - cfg.socket_dir = tmp.path().join("source-sock"); - cfg.ctl_timeout = Duration::from_secs(60); - fs::create_dir_all(&cfg.filter_out_dir).unwrap(); - fs::create_dir_all(&cfg.socket_dir).unwrap(); - Shadow::new(cfg) -} - /// CH-config TOML covering all four pgbench tables. Attnums match /// pgbench's `CREATE TABLE` order (see `pgbench --help` source, or /// `\d pgbench_accounts` post-init). @@ -244,16 +231,7 @@ async fn run_ddl_intermix( inserter_pool: usize, label: &str, ) { - if !fx::pg_available() { - tracing::warn!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - tracing::warn!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - tracing::warn!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } if !pgbench_available() { @@ -267,7 +245,7 @@ async fn run_ddl_intermix( let tmp = tempfile::tempdir().unwrap(); // 1. Source PG. - let source = make_source(&tmp, ports.source); + let source = fx::make_source(&tmp); source.initdb().expect("initdb source"); source.write_base_conf().expect("source base conf"); fx::append_source_conf(&source).expect("append source conf"); diff --git a/tests/pipeline_parallel_ddl_e2e.rs b/tests/pipeline_parallel_ddl_e2e.rs index 58624db6..4f6e1bb2 100644 --- a/tests/pipeline_parallel_ddl_e2e.rs +++ b/tests/pipeline_parallel_ddl_e2e.rs @@ -37,8 +37,7 @@ use walshadow::schema::RelName; async fn parallel_pipeline_schema_evolution_orders_after_data() { let slot = fx::Ports::alloc(); - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -203,8 +202,7 @@ async fn parallel_pipeline_schema_evolution_orders_after_data() { async fn parallel_pipeline_truncate_orders_after_data() { let slot = fx::Ports::alloc(); - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/pipeline_parallel_e2e.rs b/tests/pipeline_parallel_e2e.rs index 122e2cae..ec4c6ba1 100644 --- a/tests/pipeline_parallel_e2e.rs +++ b/tests/pipeline_parallel_e2e.rs @@ -26,16 +26,7 @@ use walshadow::schema::RelName; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn parallel_pipeline_replicates_dml() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } @@ -266,16 +257,7 @@ const META2_SQL: &str = "repeat('v2-update-', 60)"; /// the max-`_lsn` row, not NULL-skipping argMax) catches it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn parallel_pipeline_slices_multi_batch_commit() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } diff --git a/tests/runtime_config_e2e.rs b/tests/runtime_config_e2e.rs index 4e534ae3..b0410bf6 100644 --- a/tests/runtime_config_e2e.rs +++ b/tests/runtime_config_e2e.rs @@ -105,8 +105,7 @@ fn overlay_ddl_args() -> fx::DdlPipelineArgs { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn opt_in_via_config_table_replicates_new_table() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -193,8 +192,7 @@ async fn opt_in_via_config_table_replicates_new_table() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn opt_out_mid_stream_drains_and_halts() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -318,8 +316,7 @@ async fn opt_out_mid_stream_drains_and_halts() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn forward_decl_materializes_on_create_table() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -421,8 +418,7 @@ async fn forward_decl_materializes_on_create_table() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn opt_in_non_empty_backfills_pre_opt_in_rows() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -604,8 +600,7 @@ async fn opt_in_non_empty_backfills_pre_opt_in_rows() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn opt_in_then_alter_add_column_reaches_ch() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -710,8 +705,7 @@ async fn opt_in_then_alter_add_column_reaches_ch() { /// drives auto-create, not just the per-table `replicate=true` opt-in. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn auto_create_namespace_via_config_namespace() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -801,8 +795,7 @@ async fn auto_create_namespace_via_config_namespace() { /// dropped one encodes scale-0 `123`. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn column_target_type_override_reaches_projection() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -915,8 +908,7 @@ async fn column_target_type_override_reaches_projection() { /// heap rows in WAL routes those rows. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn pre_opt_in_xact_discards_post_opt_in_routes() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -1013,8 +1005,7 @@ async fn pre_opt_in_xact_discards_post_opt_in_routes() { /// Drill 9: glob rules scope tables created later #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn pattern_row_scopes_tables_by_glob() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -1105,8 +1096,7 @@ async fn pattern_row_scopes_tables_by_glob() { /// `replicate = true` creates, so the CREATE cannot fall back to the PK. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn opt_in_row_pins_order_by_and_primary_key() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -1196,8 +1186,7 @@ async fn opt_in_row_pins_order_by_and_primary_key() { /// columns of an auto-created table (plans/config.md §Destination shape). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn pattern_row_shapes_auto_created_tables() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/schema_evolution_cdc.rs b/tests/schema_evolution_cdc.rs index 325bd7cc..016b3fbc 100644 --- a/tests/schema_evolution_cdc.rs +++ b/tests/schema_evolution_cdc.rs @@ -12,8 +12,7 @@ use walshadow::mapping::NamespaceMapping; use walshadow::shadow::Shadow; fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/schema_evolution_pinned.rs b/tests/schema_evolution_pinned.rs index a59eaca6..b24fee4c 100644 --- a/tests/schema_evolution_pinned.rs +++ b/tests/schema_evolution_pinned.rs @@ -35,8 +35,7 @@ use walshadow::mapping::TableTarget; use walshadow::schema::RelName; fn skip_if_missing() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/soft_delete_cdc.rs b/tests/soft_delete_cdc.rs index 23a8f932..0374567c 100644 --- a/tests/soft_delete_cdc.rs +++ b/tests/soft_delete_cdc.rs @@ -127,8 +127,7 @@ fn winning_flag(ch: &fx::ChServer, id: i32) -> String { } fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/source_reconnect.rs b/tests/source_reconnect.rs index afe99855..f24d6be6 100644 --- a/tests/source_reconnect.rs +++ b/tests/source_reconnect.rs @@ -14,7 +14,6 @@ use std::time::Duration; use walrus::pg::backup::parse_pg_lsn; use walrus::pg::replication::conn::PgConfig; -use walrus::pg::replication::tls::{SslMode, TlsParams}; use walshadow::shadow::{Shadow, ShadowConfig}; use walshadow::source_feed::{SourceFeed, StandbyStatus}; @@ -56,19 +55,6 @@ fn append_conf(sh: &Shadow, extra: &[&str]) { } } -fn pg_cfg(sh: &Shadow) -> PgConfig { - PgConfig { - host: sh.config().socket_dir.to_str().unwrap().to_string(), - port: sh.config().port, - user: "postgres".into(), - password: None, - database: "postgres".into(), - application_name: "source-reconnect-test".into(), - sslmode: SslMode::Disable, - tls: TlsParams::default(), - } -} - fn status(lsn: u64) -> StandbyStatus { StandbyStatus::collapsed(lsn) } @@ -120,7 +106,7 @@ async fn reconnect_resumes_after_walsender_terminated() { source.start().unwrap(); let _stop = StopOnDrop(&source); - let cfg = pg_cfg(&source); + let cfg = ports::pg_cfg(&source, "source-reconnect-test"); let mut feed = SourceFeed::connect(&cfg).await.unwrap(); let ident = feed.identify_system().await.unwrap(); feed.start_physical_replication(None, ident.xlogpos, ident.timeline) @@ -178,7 +164,7 @@ async fn recycled_segment_surfaces_58p01() { source.start().unwrap(); let _stop = StopOnDrop(&source); - let cfg = pg_cfg(&source); + let cfg = ports::pg_cfg(&source, "source-reconnect-test"); let mut feed = SourceFeed::connect(&cfg).await.unwrap(); let ident = feed.identify_system().await.unwrap(); let old_lsn = ident.xlogpos; @@ -239,7 +225,7 @@ async fn slot_prevents_segment_recycle() { source.start().unwrap(); let _stop = StopOnDrop(&source); - let cfg = pg_cfg(&source); + let cfg = ports::pg_cfg(&source, "source-reconnect-test"); let slot = "walshadow_recycle_test"; let mut feed = SourceFeed::connect(&cfg).await.unwrap(); let ident = feed.identify_system().await.unwrap(); diff --git a/tests/subxact.rs b/tests/subxact.rs index be005b63..31275880 100644 --- a/tests/subxact.rs +++ b/tests/subxact.rs @@ -503,17 +503,5 @@ async fn many_subxacts_emit_assignment_record() { } fn skip_gate() -> bool { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return true; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return true; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); - return true; - } - false + !fx::requirements_available() } diff --git a/tests/system_columns_cdc.rs b/tests/system_columns_cdc.rs index d9b18fec..782d46dc 100644 --- a/tests/system_columns_cdc.rs +++ b/tests/system_columns_cdc.rs @@ -20,8 +20,7 @@ use walshadow::table_rules::{MatchKind, TableRule}; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn renamed_system_columns_and_operator_keys() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/toast_e2e.rs b/tests/toast_e2e.rs index 3de274cd..a6d63dd9 100644 --- a/tests/toast_e2e.rs +++ b/tests/toast_e2e.rs @@ -35,8 +35,7 @@ const META2_SQL: &str = "repeat('v2-update-', 60)"; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn replident_full_unchanged_toast_update() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/toast_rewrite_e2e.rs b/tests/toast_rewrite_e2e.rs index e2e4e617..757ad2cf 100644 --- a/tests/toast_rewrite_e2e.rs +++ b/tests/toast_rewrite_e2e.rs @@ -74,8 +74,7 @@ fn live_sum_sql(chunk_table: &str, max_lsn: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn vacuum_full_rewrite_and_same_xact_stash() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -464,8 +463,7 @@ async fn vacuum_full_rewrite_and_same_xact_stash() { /// fresh mirror — and the old mirror retires through the DROP lifecycle. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn alter_rewrite_link_swap_retires_old_mirror() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/toast_tombstone_e2e.rs b/tests/toast_tombstone_e2e.rs index 13fc8900..d77c3d6b 100644 --- a/tests/toast_tombstone_e2e.rs +++ b/tests/toast_tombstone_e2e.rs @@ -77,8 +77,7 @@ fn live_values_sql(chunk_table: &str, max_lsn: &str) -> String { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn tombstones_supersede_then_truncate_wipes_then_drop_retires() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/toast_truncate_drop_e2e.rs b/tests/toast_truncate_drop_e2e.rs index 35f5e912..ffb50ba0 100644 --- a/tests/toast_truncate_drop_e2e.rs +++ b/tests/toast_truncate_drop_e2e.rs @@ -79,8 +79,7 @@ fn doc_mappings() -> Vec { /// fires and the mirror leaks indefinitely. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn cold_restart_drop_retires_mirror() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -239,8 +238,7 @@ async fn cold_restart_drop_retires_mirror() { /// as the durable original, equal-version rows dedup can't arbitrate. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn drop_crash_replay_keeps_referrer_bytes() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } @@ -404,8 +402,7 @@ async fn drop_crash_replay_keeps_referrer_bytes() { /// WAL pumped at all. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn drop_retire_survives_restart_from_ledger() { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return; } diff --git a/tests/truncate.rs b/tests/truncate.rs index 4a54e27d..c9b56ed0 100644 --- a/tests/truncate.rs +++ b/tests/truncate.rs @@ -27,16 +27,7 @@ use walshadow::schema::RelName; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn truncate_removes_ch_rows() { - if !fx::pg_available() { - eprintln!("skip: no initdb on PATH"); - return; - } - if !fx::pg_basebackup_available() { - eprintln!("skip: no pg_basebackup on PATH"); - return; - } - if !fx::clickhouse_available() { - eprintln!("skip: no clickhouse binary on PATH"); + if !fx::requirements_available() { return; } diff --git a/tests/types_sweep.rs b/tests/types_sweep.rs index 693a23f4..4aaef057 100644 --- a/tests/types_sweep.rs +++ b/tests/types_sweep.rs @@ -22,8 +22,7 @@ fn col(attnum: i16, name: &str, ty: &str) -> ColumnMapping { } fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false diff --git a/tests/visibility_repair_pg.rs b/tests/visibility_repair_pg.rs new file mode 100644 index 00000000..ea393749 --- /dev/null +++ b/tests/visibility_repair_pg.rs @@ -0,0 +1,301 @@ +//! Verify visibility repair against live PostgreSQL, including descriptor +//! drift failures + +#![cfg(target_os = "linux")] + +#[path = "common/ports.rs"] +mod fx; + +use ahash::{HashSet, HashSetExt}; +use std::fs; +use std::io::Write as _; +use std::process::Command; +use std::time::Duration; + +use tokio::sync::mpsc; +use walshadow::backfill_bootstrap::seed_catalog_from_source; +use walshadow::backup_page_walk::{BackfillTuple, CatalogMap}; +use walshadow::shadow::{Shadow, ShadowConfig}; +use walshadow::source_feed::open_sql_client; +use walshadow::visibility_repair::{PendingReason, PendingSet, repair}; + +/// Coverage boundary every baseline row carries +const S: u64 = 0x0100_0000; +const APP: &str = "visibility-repair-test"; + +fn pg_available() -> bool { + Command::new("initdb") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) +} + +struct Source { + _tmp: tempfile::TempDir, + sh: Shadow, +} + +impl Drop for Source { + fn drop(&mut self) { + let _ = self.sh.stop(); + } +} + +fn start_source(sql: &str) -> Source { + let tmp = tempfile::tempdir().unwrap(); + let mut cfg = ShadowConfig::new(tmp.path().join("data"), tmp.path().join("filtered")); + cfg.port = fx::reserve_port(); + cfg.socket_dir = tmp.path().join("sock"); + cfg.ctl_timeout = Duration::from_secs(60); + fs::create_dir_all(&cfg.filter_out_dir).unwrap(); + fs::create_dir_all(&cfg.socket_dir).unwrap(); + let sh = Shadow::new(cfg); + sh.initdb().expect("initdb"); + sh.write_base_conf().expect("base conf"); + let mut f = fs::OpenOptions::new() + .append(true) + .open(sh.config().data_dir.join("postgresql.conf")) + .unwrap(); + writeln!(f, "\nwal_level = replica").unwrap(); + drop(f); + sh.start().expect("start"); + sh.apply_schema_dump(sql).expect("workload"); + Source { _tmp: tmp, sh } +} + +async fn seed(sh: &Shadow) -> CatalogMap { + let client = open_sql_client(&fx::pg_cfg(sh, APP)) + .await + .expect("sql connect"); + seed_catalog_from_source(&client).await.expect("seed") +} + +async fn drain(mut rx: mpsc::Receiver) -> Vec { + let mut out = Vec::new(); + while let Some(t) = rx.recv().await { + out.push(t); + } + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn inherited_rows_keep_their_physical_relation() { + if !pg_available() { + return; + } + let src = start_source( + "CREATE TABLE parent (id int PRIMARY KEY, body text); + CREATE TABLE child () INHERITS (parent); + INSERT INTO parent VALUES (1, 'parent'); + INSERT INTO child VALUES (2, 'child');", + ); + let catalog = seed(&src.sh).await; + let parent = catalog + .descriptors() + .find(|d| &*d.rel_name.name == "parent") + .unwrap(); + let pending = PendingSet::toast_capable(&catalog).scoped_to( + [(parent.rfn.db_node, parent.rfn.rel_node)] + .into_iter() + .collect(), + ); + let (tx, rx) = mpsc::channel(4); + let collect = tokio::spawn(drain(rx)); + let stats = repair( + &pending, + &catalog, + &HashSet::new(), + &fx::pg_cfg(&src.sh, APP), + S, + &tx, + ) + .await + .unwrap(); + drop(tx); + let rows = collect.await.unwrap(); + assert_eq!(stats.rows, 1); + assert_eq!(rows[0].rfn, parent.rfn); + assert!(matches!( + rows[0].columns[0], + Some(walshadow::heap_decoder::ColumnValue::Int4(1)) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn row_security_cannot_silently_filter_repair() { + if !pg_available() { + return; + } + let src = start_source( + "CREATE TABLE t (id int PRIMARY KEY, body text); + INSERT INTO t VALUES (1, 'visible'), (2, 'hidden'); + CREATE ROLE repl LOGIN REPLICATION; + GRANT SELECT ON t TO repl; + ALTER TABLE t ENABLE ROW LEVEL SECURITY; + CREATE POLICY visible ON t FOR SELECT TO repl USING (id=1);", + ); + let catalog = seed(&src.sh).await; + let pending = PendingSet::toast_capable(&catalog); + let mut pg = fx::pg_cfg(&src.sh, APP); + pg.user = "repl".into(); + let (tx, mut rx) = mpsc::channel(4); + let err = repair(&pending, &catalog, &HashSet::new(), &pg, S, &tx) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("row-level security"), "{err:#}"); + drop(tx); + assert!(rx.recv().await.is_none()); +} + +/// Read TOAST-owning relation whole through PostgreSQL +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn toast_capable_relation_is_read_whole_at_the_coverage_lsn() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + let src = start_source( + "CREATE TABLE public.t (id int4 PRIMARY KEY, body text NOT NULL) \ + WITH (autovacuum_enabled = false);\n\ + ALTER TABLE public.t ALTER COLUMN body SET STORAGE EXTERNAL;\n\ + INSERT INTO public.t SELECT g, repeat('body-'||g::text||'---', 700) \ + FROM generate_series(1, 6) g;\n\ + DELETE FROM public.t WHERE id > 4;\n\ + CREATE TABLE public.fixed (id int4 PRIMARY KEY, n int8 NOT NULL);\n\ + INSERT INTO public.fixed SELECT g, g FROM generate_series(1, 3) g;\n", + ); + let catalog = seed(&src.sh).await; + let pending = PendingSet::toast_capable(&catalog); + // `fixed` has no varlena column, so PostgreSQL gave it no toast relation + assert_eq!( + pending.len(), + 1, + "only the text-bearing relation is pending" + ); + assert_eq!(pending.count_for(PendingReason::ExternalToast), 1); + + let (tx, rx) = mpsc::channel(64); + let collect = tokio::spawn(drain(rx)); + let stats = repair( + &pending, + &catalog, + &HashSet::new(), + &fx::pg_cfg(&src.sh, APP), + S, + &tx, + ) + .await + .expect("repair"); + drop(tx); + let rows = collect.await.unwrap(); + + assert_eq!(stats.relations, 1); + assert_eq!(stats.rows, 4, "the deleted versions are not visible"); + assert_eq!(rows.len(), 4); + // Baseline uses coverage LSN + assert!(rows.iter().all(|r| r.source_lsn == S)); + assert!( + stats.p_hi > S, + "p_hi is a frontier, sampled after the reads" + ); + // Bodies came back detoasted, so nothing here consulted a chunk mirror + let body = rows[0].columns[1].as_ref().expect("body column"); + assert!( + format!("{body:?}").contains("body-"), + "external value arrived inline: {body:?}" + ); +} + +/// Skip relations excluded from initial load +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn opted_out_relation_is_skipped_not_scanned() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + let src = start_source( + "CREATE TABLE public.t (id int4 PRIMARY KEY, body text NOT NULL);\n\ + INSERT INTO public.t VALUES (1, 'one');\n", + ); + let catalog = seed(&src.sh).await; + let pending = PendingSet::toast_capable(&catalog); + let skip: HashSet<_> = catalog.descriptors().map(|d| d.rel_name.clone()).collect(); + + let (tx, rx) = mpsc::channel(4); + let collect = tokio::spawn(drain(rx)); + let stats = repair(&pending, &catalog, &skip, &fx::pg_cfg(&src.sh, APP), S, &tx) + .await + .expect("repair"); + drop(tx); + + assert_eq!(stats.skipped, 1); + assert_eq!(stats.relations, 0); + assert_eq!(stats.rows, 0); + assert!(collect.await.unwrap().is_empty()); +} + +/// Seed the catalog off a baseline relation, mutate the source, then require +/// repair to reject the drift naming every needle in `expected` +async fn assert_repair_rejects(change: &str, expected: &[&str]) { + let src = start_source( + "CREATE TABLE public.t (id int4 PRIMARY KEY, body text NOT NULL);\n\ + INSERT INTO public.t SELECT g, 'row-'||g::text FROM generate_series(1, 4) g;\n", + ); + let catalog = seed(&src.sh).await; + let pending = PendingSet::toast_capable(&catalog); + src.sh.apply_schema_dump(change).expect("source change"); + + let (tx, _rx) = mpsc::channel(4); + let err = repair( + &pending, + &catalog, + &HashSet::new(), + &fx::pg_cfg(&src.sh, APP), + S, + &tx, + ) + .await + .expect_err("drifted relation is not repairable"); + let msg = format!("{err:#}"); + for needle in expected { + assert!(msg.contains(needle), "want {needle:?} in {msg}"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rewritten_relation_fails_the_pass() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + // Rewrite rotates filenode + assert_repair_rejects( + "VACUUM FULL public.t;\n", + &["rewritten inside the backup window"], + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropped_relation_fails_the_pass() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + assert_repair_rejects("DROP TABLE public.t;\n", &["gone from the source"]).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn added_column_fails_the_pass() { + if !pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + // No rewrite, so the filenode still matches; the shape does not. The + // report names the new column + assert_repair_rejects( + "ALTER TABLE public.t ADD COLUMN extra int4;\n", + &["changed inside the backup window", "extra"], + ) + .await; +} diff --git a/tests/weird_identifiers.rs b/tests/weird_identifiers.rs index 509ab763..c4a370ac 100644 --- a/tests/weird_identifiers.rs +++ b/tests/weird_identifiers.rs @@ -92,8 +92,7 @@ fn create_ch_dests(ch: &fx::ChServer) { } fn skip_gate() -> bool { - if !fx::pg_available() || !fx::pg_basebackup_available() || !fx::clickhouse_available() { - eprintln!("skip: missing initdb / pg_basebackup / clickhouse"); + if !fx::requirements_available() { return true; } false From 953c77a70044ae25292936c21749517fd9df6daa Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:16:18 +0000 Subject: [PATCH 2/2] replace spaghetti architecture diagrams --- README.md | 1 + architecture/README.md | 147 ++--- architecture/bootstrap.dot | 162 ----- architecture/bootstrap.svg | 677 +++------------------ architecture/catalog.svg | 60 ++ architecture/decoder.dot | 149 ----- architecture/decoder.svg | 594 ------------------- architecture/emitter.dot | 145 ----- architecture/emitter.svg | 477 --------------- architecture/filter.dot | 132 ----- architecture/filter.svg | 433 -------------- architecture/internals.dot | 177 ------ architecture/internals.svg | 813 -------------------------- architecture/ops.dot | 124 ---- architecture/ops.svg | 403 ------------- architecture/oracle.dot | 126 ---- architecture/oracle.svg | 472 --------------- architecture/overview.dot | 44 -- architecture/overview.svg | 246 +++----- architecture/palette.md | 77 --- architecture/recovery.svg | 71 +++ architecture/shadow.dot | 100 ---- architecture/shadow.svg | 349 ----------- architecture/shadow_communication.dot | 84 --- architecture/shadow_communication.svg | 267 --------- architecture/source.dot | 119 ---- architecture/source.svg | 436 -------------- architecture/timeline_bootstrap.dot | 120 ---- architecture/timeline_bootstrap.svg | 482 --------------- architecture/timeline_restart.dot | 98 ---- architecture/timeline_restart.svg | 450 -------------- architecture/timeline_streaming.dot | 151 ----- architecture/timeline_streaming.svg | 582 ------------------ architecture/toast.dot | 108 ---- architecture/toast.svg | 423 -------------- architecture/values.svg | 71 +++ architecture/workers.svg | 105 ++++ architecture/xact.dot | 126 ---- architecture/xact.svg | 400 ------------- plans/INDEX.md | 21 +- plans/TOAST.md | 2 +- plans/bootstrap.md | 4 +- plans/decoder.md | 2 +- plans/emitter.md | 2 +- plans/filter.md | 2 +- plans/ops.md | 2 +- plans/oracle.md | 2 +- plans/shadow.md | 4 +- plans/source.md | 2 +- plans/xact.md | 2 +- 50 files changed, 526 insertions(+), 9520 deletions(-) delete mode 100644 architecture/bootstrap.dot create mode 100644 architecture/catalog.svg delete mode 100644 architecture/decoder.dot delete mode 100644 architecture/decoder.svg delete mode 100644 architecture/emitter.dot delete mode 100644 architecture/emitter.svg delete mode 100644 architecture/filter.dot delete mode 100644 architecture/filter.svg delete mode 100644 architecture/internals.dot delete mode 100644 architecture/internals.svg delete mode 100644 architecture/ops.dot delete mode 100644 architecture/ops.svg delete mode 100644 architecture/oracle.dot delete mode 100644 architecture/oracle.svg delete mode 100644 architecture/overview.dot delete mode 100644 architecture/palette.md create mode 100644 architecture/recovery.svg delete mode 100644 architecture/shadow.dot delete mode 100644 architecture/shadow.svg delete mode 100644 architecture/shadow_communication.dot delete mode 100644 architecture/shadow_communication.svg delete mode 100644 architecture/source.dot delete mode 100644 architecture/source.svg delete mode 100644 architecture/timeline_bootstrap.dot delete mode 100644 architecture/timeline_bootstrap.svg delete mode 100644 architecture/timeline_restart.dot delete mode 100644 architecture/timeline_restart.svg delete mode 100644 architecture/timeline_streaming.dot delete mode 100644 architecture/timeline_streaming.svg delete mode 100644 architecture/toast.dot delete mode 100644 architecture/toast.svg create mode 100644 architecture/values.svg create mode 100644 architecture/workers.svg delete mode 100644 architecture/xact.dot delete mode 100644 architecture/xact.svg diff --git a/README.md b/README.md index 19c6d7d4..11738263 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ PostgreSQL major version ## Documentation - [Documentation index](docs/README.md) +- [Architecture](architecture/README.md) - [Configuration](docs/configuration.md) - [Table selection](docs/table-selection.md) - [Destination tables](docs/destination-tables.md) diff --git a/architecture/README.md b/architecture/README.md index 70f584b4..1cbdf140 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -1,123 +1,56 @@ -### 1. Overview — Postgres → walshadow → ClickHouse +# Architecture -High-level pipeline. Shadow PG runs as catalog-replay sidecar fed by -walshadow's walsender; filtered segments under `out/` serve as archive -fallback. CH rows buffer across xacts and seal as complete INSERTs -(budget / deadline) shipped over an N-connection inserter pool; -walshadow pushes DDL through its own CH connection. +walshadow consumes PostgreSQL physical WAL, replays filtered WAL in a shadow +PostgreSQL process, and reconstructs committed rows for ClickHouse -![overview](overview.svg) +## Streaming topology -### 2. Internals — pipeline, taps & caches +![Streaming topology: original records feed a bounded queue and transaction buffer; filtered WAL feeds shadow; catalog capture supplies descriptor history and schema events to row processing](overview.svg) -Hot path runs top→bottom; ancillaries (catalog cache, walsender server, -disk artifacts) sit off to the right with `constraint=false` edges so -they don't pull the main column off axis. `QueueingRecordSink` between -fan-out and decoder keeps the decoder's `wait_for_replay` off the pump -task so the walsender wire never stalls behind it. CH and metrics-only -runs now share one transaction and acknowledgement pipeline. CH mode -adds inserter pool and DDL connection. TOAST side path persists chunks, -serves older values, and applies lifecycle barriers. +`WalStream` retains original records for decoding and rewrites user-table +records to no-ops for shadow replay. Shadow receives filtered bytes through +walshadow's sender, with local segments as archive fallback -![internals](internals.svg) +`CatalogCapture` holds publication at schema boundaries, reads shadow at an +exact replay position, persists descriptors, and attaches `SchemaEvent` to +`XactBuffer`. Queued row processing uses that history when decoding and +planning committed transactions -### 3. Shadow communication — three channels +## Commit pipeline -How walshadow talks to shadow PG: ① libpq catalog queries, ② walsender -wire at record cadence, ③ `restore_command` archive fallback, plus the -one-shot BASE_BACKUP land for greenfield bootstrap. Schema-event flow -derives off channel ① (cache miss → diff → `SchemaEvent` → -`DdlApplicator` → CH) and stays inside walshadow. +![Commit pipeline: bounded DecodeJob queue fans out to M workers, rows merge through one batcher, InsertBatch queue fans out to N inserters, and separate Register, Placed and Acked events advance a contiguous watermark](workers.svg) -![shadow communication](shadow_communication.svg) +`BufferingDecoderSink` and `ReorderSink` share one record-queue worker +`[ch].decoder_pool_size` and `[ch].inserter_pool_size` size downstream pools +Each inserter owns a ClickHouse connection and can take any sealed batch -### 4. Bootstrap timeline — greenfield in five phases +Sequence numbers identify work slices, not necessarily whole transactions +Only a commit's final slice publishes its LSN, after all earlier work finishes +Bounded queues and a shared payload budget limit work in flight; transaction +and plan data can spill to disk -Catalog seed → BASE_BACKUP pump → drain to CH → shadow handoff → WAL -streaming. A concurrent WAL leg ships backup-window commits at their real -`_lsn`, above walked rows. Bootstrap waits for CH writes, then uses backup end -as restart point. First status update saves it in `manifest.toml`. +## Related paths -![bootstrap timeline](timeline_bootstrap.svg) - -### 5. Streaming timeline — one record's journey - -Steady-state hot path, top→bottom. Bytes path (③→④) stays on the pump -task; decoder path (③→④'→⑤→⑥) crosses `QueueingRecordSink` so it can -wait on shadow without parking the wire. ⑥ is the parallel pipeline: -reorder assigns a dense seq per commit, decode pool routes rows, the -batcher buffers per table across xacts and seals one complete INSERT -per budget/deadline window, inserter pool ships N in flight; the ack -collector advances only after every earlier commit is durable. Status -loop saves a conservative restart point in `manifest.toml`, then shares -that saved point with cleanup tasks. Reorder persists TOAST changes -before commit publication; decode uses current transaction first, then -mirror history. - -![streaming timeline](timeline_streaming.svg) - -### 6. Restart timelines — three scenarios +| Diagram | Scope | Implementation | +|---|---|---| +| [Catalog capture and DDL](catalog.svg) | Capture on descriptor-log miss, pinned SCAN, persistence, placement/flush/durability barrier | [capture](../src/source/catalog_capture.rs), [reorder](../src/emit/pipeline/reorder.rs) | +| [TOAST and type conversion](values.svg) | Transaction chunks, versioned TOAST mirrors, per-batch shadow conversion, Native block assembly | [resolver](../src/toast/resolver.rs), [oracle](../src/ops/oracle.rs), [inserter](../src/emit/pipeline/inserter.rs) | +| [Bootstrap](bootstrap.svg) | Backup fan-out, visibility gate, concurrent WAL window, separate insert tails, handoff | [backup](../src/backfill/backfill_bootstrap.rs), [window](../src/backfill/bootstrap_window.rs), [daemon](../src/bin/stream.rs) | +| [Restart and cleanup](recovery.svg) | Progress inputs, persisted restart floor, descriptor GC, TOAST retirement, source feedback | [manifest](../src/source/manifest.rs), [status loop](../src/bin/stream.rs) | -Side-by-side columns: A. clean SIGTERM, B. kill -9 mid-stream -(validated by `tests/kill_restart.rs` drill), C. WAL overflow → -source/archive/source fallback, else operator resolution. Includes -`manifest.toml` restart state and source identity. `toast_retires.toml` -survives transaction-spill cleanup and flushes safe mirror retirements at -startup. +Streaming wiring lives in [stream.rs](../src/bin/stream.rs), queue ownership +in [queueing_record_sink.rs](../src/source/queueing_record_sink.rs), and pool +assembly in [pipeline/mod.rs](../src/emit/pipeline/mod.rs) -![restart timelines](timeline_restart.svg) +## Diagram sources -## Component diagrams +SVGs are editable source. Rectangles identify components, dashed enclosures +identify processes or worker groups, narrow bars identify queues, and cylinders +identify stored state. Solid arrows carry data; dashed arrows carry progress +or control. Labels name messages, protocols, or state transferred -Focused views for components with load-bearing topology. Embedded inline -in matching plan docs. Render alongside six system views above. +Use dark colors: warm neutral backgrounds and text, blue data paths, orange control +paths, green catalog paths, magenta stored state, and yellow ClickHouse borders -| component | source | embedded in | -|---|---|---| -| filter | [`filter.dot`](filter.dot) | [`plans/filter.md`](../plans/filter.md) | -| source | [`source.dot`](source.dot) | [`plans/source.md`](../plans/source.md) | -| shadow | [`shadow.dot`](shadow.dot) | [`plans/shadow.md`](../plans/shadow.md) | -| decoder | [`decoder.dot`](decoder.dot) | [`plans/decoder.md`](../plans/decoder.md) | -| xact | [`xact.dot`](xact.dot) | [`plans/xact.md`](../plans/xact.md) | -| TOAST | [`toast.dot`](toast.dot) | [`plans/TOAST.md`](../plans/TOAST.md) | -| emitter | [`emitter.dot`](emitter.dot) | [`plans/emitter.md`](../plans/emitter.md) | -| bootstrap | [`bootstrap.dot`](bootstrap.dot) | [`plans/bootstrap.md`](../plans/bootstrap.md) | -| ops | [`ops.dot`](ops.dot) | [`plans/ops.md`](../plans/ops.md) | -| oracle | [`oracle.dot`](oracle.dot) | [`plans/oracle.md`](../plans/oracle.md) | - -## Regenerating a diagram - -Each `.dot` carries its own regeneration spec as a header comment -(sources of truth, `plans/` section subsumed, quality bar). Shared style -— palette, edge channels, legend conventions — lives in -[`palette.md`](palette.md). - -To regenerate `architecture/.svg`: -1. read [`palette.md`](palette.md) for shared style invariants -2. read the regen-spec header in `.dot` (sources of truth, subsumes, quality bar) -3. read `plans/.md` for current implementation truth, plus the cited `src/` files as accuracy anchor -4. edit `.dot`, render (below), read the png, iterate until the header quality bar passes -5. if the `.svg` path changed, update the `plans/.md` embed - -System-level diagrams (overview, internals, shadow_communication, -timeline_*) carry no per-comp spec — stable and visually saturated. Add -one only on the next material rewrite. - -## Render - -```sh -for f in *.dot; do dot -Tsvg "$f" -o "${f%.dot}.svg"; dot -Tpng "$f" -o "${f%.dot}.png"; done -``` - -## Key references - -| diagram detail | source | -|---|---| -| catalog-event channel + CH DDL applicator | [`plans/shadow.md`](../plans/shadow.md), [`plans/emitter.md`](../plans/emitter.md) | -| atomic-seal INSERT, TRUNCATE, subxact rollback, apply-lag | [`plans/emitter.md`](../plans/emitter.md), [`plans/xact.md`](../plans/xact.md), [`plans/ops.md`](../plans/ops.md) | -| `QueueingRecordSink`, pump ↔ decoder decoupling | [`plans/source.md`](../plans/source.md) | -| streaming-fed shadow | [`plans/shadow.md`](../plans/shadow.md), [`plans/source.md`](../plans/source.md) | -| greenfield bootstrap | [`plans/bootstrap.md`](../plans/bootstrap.md) | -| saved restart manifest | [`plans/ops.md`](../plans/ops.md) | -| xact buffer + disk spill | [`plans/xact.md`](../plans/xact.md) | -| TOAST mirror, fetch, bootstrap, rewrite, retirement | [`plans/TOAST.md`](../plans/TOAST.md) | +Keep diagrams here and embed them from plans. Check component names and +connections against linked source, then render at full and README widths diff --git a/architecture/bootstrap.dot b/architecture/bootstrap.dot deleted file mode 100644 index 2162f224..00000000 --- a/architecture/bootstrap.dot +++ /dev/null @@ -1,162 +0,0 @@ -// walshadow — BASE_BACKUP fan-out (component view) -// Differentiated from timeline_bootstrap.dot: that one walks the -// 5 phases top→bottom in time; this one freezes phases 1-4 and -// exposes the MultiplexSink fan-out structure — BackupSource trait -// + two impls, per-file routing, parallel ShadowDataDir + PageWalkSink -// rails, drain into the shared insert tail (same unit as streaming). -// -// regeneration spec: -// sources of truth: plans/bootstrap.md · src/backfill/{backup_source_direct,backup_source_object_store,backup_page_walk,backup_source}.rs · src/emit/pipeline/{bootstrap,tail}.rs -// subsumes: plans/bootstrap.md § "MultiplexSink" / fan-out -// differentiates: timeline_bootstrap.dot walks the 5-phase TIMELINE; this freezes phases 1-4 and exposes the FAN-OUT structure (BackupSource impls, MultiplexSink, simultaneous shadow-dir + CH writes) -// quality bar: -// - "both" fan-out from classify visible (two edges, not one merged) -// - CatalogMap lookup edge clearly secondary (dashed, constraint=false) -// - bootstrap CH path reads as the SAME shared insert tail streaming uses (annotated "no DdlApplicator") -// shared style: palette.md -digraph bootstrap { - rankdir=TB; - compound=true; - newrank=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow greenfield bootstrap — BASE_BACKUP fan-out (MultiplexSink, parallel rails)", fontsize=14, splines=spline, nodesep=0.45, ranksep=0.6, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ External actors (top) ════════ - src [label="source PG\nreplication slot", fillcolor="#3D3D54", shape=cylinder]; - s3 [label="object store\nwal-g layout\n(DynStorage)", fillcolor="#3D3D54", shape=cylinder]; - { rank=same; src; s3; } - - // ════════ Catalog seed sidecar (off-pump) ════════ - subgraph cluster_seed { - label="catalog seed — REPEATABLE READ sidecar, runs once before BASE_BACKUP"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - seedq [label="seed_in_snapshot\nSELECT pg_class / pg_attribute /\npg_type / pg_index WHERE oid≥16384", fillcolor="#4D4D28"]; - catmap [label="CatalogMap\n(db_node, rel_node) → RelDescriptor\nsnapshot, no replay gate", fillcolor="#4D4D28", shape=parallelogram]; - seedq -> catmap; - } - - // ════════ BackupSource trait + two impls ════════ - subgraph cluster_source { - label="BackupSource — async fn run(data_dir, Arc>) → (StartInfo, EndInfo)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - direct [label="BackupSourceDirect\nwal-rus run_base_backup\ntokio_tar over ChannelReader", fillcolor="#4D3A28"]; - obj [label="BackupSourceObjectStore\nfetch_sentinel + list_tar_parts\nbuffer_unordered, pg_control barrier", fillcolor="#4D3A28"]; - ev [label="per-file events into sink\nstart → begin → chunk* → end → finish\nFileMeta {path, size, mode, kind}", fillcolor="#4D3A28", shape=parallelogram]; - { rank=same; direct; obj; } - direct -> ev; - obj -> ev; - } - src -> direct [label="BASE_BACKUP\n+ pg_export_snapshot()", color="#A1A9CC", dir=both, arrowtail=open, penwidth=2]; - s3 -> obj [label="GET tar parts", color="#A1A9CC", penwidth=2]; - src -> seedq [label="libpq sidecar", color="#CBA85E", dir=both, arrowtail=open]; - - // ════════ Orchestrator + MultiplexSink dispatcher ════════ - orch [label="backfill_bootstrap orchestrator\nspawn_greenfield_bootstrap\nholds Arc>>", fillcolor="#4D3A28"]; - mux [label="MultiplexSink (impl BackupSink, async)\nbegin(meta) → DiskLanderSink.classify():\n Keep → lander only\n SkipDenylist → lander Skip (dir entry kept as empty dir)\n SkipUserHeap → tap.begin (Tap | Skip)\nchunk / end route to chosen sink; finish observed by both", - fillcolor="#4D3340", width=4.6]; - ev -> orch [label="drives via run(sink)", color="#A1A9CC"]; - orch -> mux [label="install sink"]; - - // ════════ Left rail — shadow data dir landing ════════ - subgraph cluster_land { - label="rail A — catalog + system files (Keep)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - lander [label="DiskLanderSink\nKeep: global/, pg_xact/, pg_multixact/,\n pg_filenode.map, tablespace_map,\n pg_control, backup_label,\n pg_tblspc/ symlinks,\n base// + CatalogFilenodes\nSkipUserHeap: fn ≥ 16384 (rerouted by mux)", fillcolor="#4D3340"]; - write [label="write_kept\nFile / Dir / Symlink\nsync_data on close\npg_control lands last (barrier)", fillcolor="#4D3340"]; - ddir [label="shadow data_dir\n+ postgresql.auto.conf\n+ standby.signal\n+ restore_command", fillcolor="#4D3850", shape=note]; - lander -> write [color="#6E6963"]; - write -> ddir [color="#6E6963", style=dashed, label="fsync"]; - } - - // ════════ Right rail — page-walk → CH ════════ - subgraph cluster_walk { - label="rail B — user heap (Tap, 2A page-walk decoder)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - pwsink [label="PageWalkSink\nparse_base_path → rfn ≥ 16384\naccumulate 8 KiB at a time", fillcolor="#4D4128"]; - walker [label="PageWalker::walk_page\npd_lower/pd_upper bounds-check\nfor each LP_NORMAL ItemIdData slot\n decode_on_page_tuple\nreshape HeapTupleHeaderData →\n xl_heap_header + bitmap + cols", fillcolor="#4D4128"]; - dec [label="heap_decoder::decode_block_data\nshared with WAL hot path\nmain tuple → BackfillTuple\npg_toast tuple → columns + on-page TID\n(no FPI replay on backup pages)", fillcolor="#4D4128"]; - bft [label="BackfillTuple\n{rfn, xid, source_lsn = start_lsn,\n columns}", fillcolor="#4D4128", shape=parallelogram]; - queue [label="bounded mpsc (cap 256)\nchunk() awaits a free slot —\nslow drain backpressures the pump", fillcolor="#4D4128", shape=parallelogram]; - drain [label="pipeline::bootstrap::drain\ntoast tuple → batch ToastRow\nmain ExternalToast → Deferred\nflush mirror rows, then resolve deferred\nmap + route main rows to tail\none ack seq per rfn flip", fillcolor="#4D4128"]; - - pwsink -> walker -> dec -> bft -> queue -> drain; - } - - // MultiplexSink → rails (lander left, walker right) — same rank pulls them side-by-side - mux -> lander [label="Keep /\nSkipDenylist"]; - mux -> pwsink [label="SkipUserHeap →\ntap.begin (Tap)"]; - { rank=same; lander; pwsink; } - - // ════════ Shared insert tail (same unit as streaming) ════════ - subgraph cluster_emit { - label="shared insert tail (pipeline/tail.rs) — same batcher + inserter pool + ack collector streaming uses; no DdlApplicator (descriptors frozen at snapshot)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - batch [label="InsertBatcher\nTableEncoder per table\nNative columns + _lsn=start_lsn /\n_xid / _commit_ts / _is_deleted=0\nbudget + deadline seal", fillcolor="#5D4628"]; - ins [label="inserter pool ×N\none complete INSERT per sealed batch\nsend_with_retry + reconnect", fillcolor="#5D4628"]; - ackc [label="ack collector\nRegister(seq, start_lsn) / Placed / Acked\nwait_through(K) = all durable", fillcolor="#4D3A28", shape=note]; - batch -> ins; - ins -> ackc [style=dotted, color="#b380b0", label="Acked"]; - } - drain -> batch [color="#BF8C5F", penwidth=2, style=dashed, label="BatcherMsg::Row"]; - drain -> ackc [style=dotted, color="#b380b0", constraint=false, label="Register / Placed\n(per rfn)"]; - catmap -> drain [color="#CBA85E", style=dashed, constraint=false, label="get(db_node, rel_node)"]; - - ch [label="ClickHouse\nReplacingMergeTree(_lsn)", fillcolor="#4D4128", shape=cylinder]; - toastch [label="ClickHouse TOAST mirrors\npg_toast_\nTID-keyed births + tombstones", fillcolor="#4D4128", shape=cylinder]; - ins -> ch [color="#BF8C5F", style=dashed, penwidth=2, label="Native\n(bootstrap)"]; - drain -> toastch [color="#BF8C5F", style=dashed, penwidth=2, label="ToastResolver::put\nbefore deferred fetch"]; - - // ════════ Handoff at end_lsn ════════ - subgraph cluster_handoff { - label="handoff — finish initial load, start streaming"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - out [label="BootstrapOutcome\n{start_lsn, end_lsn,\n DiskLanderStats, PageWalkStats}\ntail.finish: FlushAll →\nwait_through(next_seq) → drain cascade", fillcolor="#4D3A28"]; - seed [label="use backup end as\nnew restart point", fillcolor="#4D3A28"]; - manifest [label="manifest.toml\nsaved restart state", fillcolor="#4D3850", shape=note]; - feed [label="SourceFeed open\nSTART_REPLICATION\nPHYSICAL ", fillcolor="#4D3A28"]; - shd [label="shadow PG\npostmaster + walreceiver\nbegin replay at end_lsn", fillcolor="#3D4128", shape=cylinder]; - - out -> seed; - seed -> feed; - feed -> manifest [color="#b380b0", style=dotted, label="save on first\nstatus update"]; - feed -> shd [color="#BD8183", style=dashed, label="walsender wire\n(steady-state)"]; - } - orch -> out [label="finish()\nend_lsn", color="#A1A9CC", constraint=false]; - ddir -> shd [color="#6E6963", style=dashed, label="shadow boots\nfrom data_dir"]; - - // ════════ Legend — anchored off ch, rank=sink so it floats to the right ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - - - - - - -
node fill — role
source PG / object store ingress
orchestrator / BackupSource impls / ack collector
MultiplexSink + DiskLanderSink (rail A)
PageWalkSink + heap decoder (rail B)
CatalogMap seed
shared insert tail (batcher + inserters)
on-disk artifact
shadow Postgres (post-handoff)
edge colour
━━replication protocol (BASE_BACKUP, START_REPLICATION)
┄┄walsender wire (post-handoff)
━━libpq catalog query / CatalogMap lookup
┄┄CH Native (bootstrap, via shared tail)
···manifest durability / ack events
┄┄filesystem (data_dir land, shadow boot)
file routing (MultiplexSink)
catalog filenode
(fn < 16384 ∪ whitelist)		rail A only
global/, pg_xact/, pg_control,
tablespace_map, conf files		rail A only
pg_replslot/, pg_stat_tmp/,
pg_logical/, pgsql_tmp/, temp_*		Skip (denylist;
dir entry kept as empty dir)
user heap
base/<db>/<fn ≥ 16384>		rail B only
(never lands on shadow)
pg_toast_<relid>rail B decodes on-page TIDs;
mirror put precedes deferred resolution
- >]; - toastch -> legend [style=invis]; -} diff --git a/architecture/bootstrap.svg b/architecture/bootstrap.svg index a33e27f5..1338fe6f 100644 --- a/architecture/bootstrap.svg +++ b/architecture/bootstrap.svg @@ -1,604 +1,75 @@ - - - - - - -bootstrap - -walshadow greenfield bootstrap — BASE_BACKUP fan-out (MultiplexSink, parallel rails) - -cluster_seed - -catalog seed — REPEATABLE READ sidecar, runs once before BASE_BACKUP - - -cluster_source - -BackupSource — async fn run(data_dir, Arc<Mutex<dyn BackupSink>>) → (StartInfo, EndInfo) - - -cluster_land - -rail A — catalog + system files (Keep) - - -cluster_walk - -rail B — user heap (Tap, 2A page-walk decoder) - - -cluster_emit - -shared insert tail (pipeline/tail.rs) — same batcher + inserter pool + ack collector streaming uses; no DdlApplicator (descriptors frozen at snapshot) - - -cluster_handoff - -handoff — finish initial load, start streaming - - - -src - - -source PG -replication slot - - - -seedq - -seed_in_snapshot -SELECT pg_class / pg_attribute / -pg_type / pg_index WHERE oid≥16384 - - - -src->seedq - - - -libpq sidecar - - - -direct - -BackupSourceDirect -wal-rus run_base_backup -tokio_tar over ChannelReader - - - -src->direct - - - -BASE_BACKUP -+ pg_export_snapshot() - - - -s3 - - -object store -wal-g layout -(DynStorage) - - - -obj - -BackupSourceObjectStore -fetch_sentinel + list_tar_parts -buffer_unordered, pg_control barrier - - - -s3->obj - - -GET tar parts - - - -catmap - -CatalogMap -(db_node, rel_node) → RelDescriptor -snapshot, no replay gate - - - -seedq->catmap - - - - - -drain - -pipeline::bootstrap::drain -toast tuple → batch ToastRow -main ExternalToast → Deferred -flush mirror rows, then resolve deferred -map + route main rows to tail -one ack seq per rfn flip - - - -catmap->drain - - -get(db_node, rel_node) - - - -ev - -per-file events into sink -start → begin → chunk* → end → finish -FileMeta {path, size, mode, kind} - - - -direct->ev - - - - - -obj->ev - - - - - -orch - -backfill_bootstrap orchestrator -spawn_greenfield_bootstrap -holds Arc<Mutex<MultiplexSink<PageWalkSink>>> - - - -ev->orch - - -drives via run(sink) - - - -mux - -MultiplexSink (impl BackupSink, async) -begin(meta) → DiskLanderSink.classify(): -  Keep         → lander only -  SkipDenylist → lander Skip (dir entry kept as empty dir) -  SkipUserHeap → tap.begin (Tap | Skip) -chunk / end route to chosen sink; finish observed by both - - - -orch->mux - - -install sink - - - -out - -BootstrapOutcome -{start_lsn, end_lsn, - DiskLanderStats, PageWalkStats} -tail.finish: FlushAll → -wait_through(next_seq) → drain cascade - - - -orch->out - - -finish() -end_lsn - - - -lander - -DiskLanderSink -Keep: global/, pg_xact/, pg_multixact/, -  pg_filenode.map, tablespace_map, -  pg_control, backup_label, -  pg_tblspc/<oid> symlinks, -  base/<db>/<fn<16384> + CatalogFilenodes -SkipUserHeap: fn ≥ 16384 (rerouted by mux) - - - -mux->lander - - -Keep / -SkipDenylist - - - -pwsink - -PageWalkSink -parse_base_path → rfn ≥ 16384 -accumulate 8 KiB at a time - - - -mux->pwsink - - -SkipUserHeap → -tap.begin (Tap) - - - -write - -write_kept -File / Dir / Symlink -sync_data on close -pg_control lands last (barrier) - - - -lander->write - - - - - -ddir - - - -shadow data_dir -+ postgresql.auto.conf -+ standby.signal -+ restore_command - - - -write->ddir - - -fsync - - - -shd - - -shadow PG -postmaster + walreceiver -begin replay at end_lsn - - - -ddir->shd - - -shadow boots -from data_dir - - - -walker - -PageWalker::walk_page -pd_lower/pd_upper bounds-check -for each LP_NORMAL ItemIdData slot -  decode_on_page_tuple -reshape HeapTupleHeaderData → -  xl_heap_header + bitmap + cols - - - -pwsink->walker - - - - - -dec - -heap_decoder::decode_block_data -shared with WAL hot path -main tuple → BackfillTuple -pg_toast tuple → columns + on-page TID -(no FPI replay on backup pages) - - - -walker->dec - - - - - -bft - -BackfillTuple -{rfn, xid, source_lsn = start_lsn, - columns} - - - -dec->bft - - - - - -queue - -bounded mpsc (cap 256) -chunk() awaits a free slot — -slow drain backpressures the pump - - - -bft->queue - - - - - -queue->drain - - - - - -batch - -InsertBatcher -TableEncoder per table -Native columns + _lsn=start_lsn / -_xid / _commit_ts / _is_deleted=0 -budget + deadline seal - - - -drain->batch - - -BatcherMsg::Row - - - -ackc - - - -ack collector -Register(seq, start_lsn) / Placed / Acked -wait_through(K) = all durable - - - -drain->ackc - - -Register / Placed -(per rfn) - - - -toastch - - -ClickHouse TOAST mirrors -pg_toast_<relid> -TID-keyed births + tombstones - - - -drain->toastch - - -ToastResolver::put -before deferred fetch - - - -ins - -inserter pool ×N -one complete INSERT per sealed batch -send_with_retry + reconnect - - - -batch->ins - - - - - -ins->ackc - - -Acked - - - -ch - - -ClickHouse -ReplacingMergeTree(_lsn) - - - -ins->ch - - -Native -(bootstrap) - - - -legend - - - -node fill — role - - - -source PG / object store ingress - - - -orchestrator / BackupSource impls / ack collector - - - -MultiplexSink + DiskLanderSink (rail A) - - - -PageWalkSink + heap decoder (rail B) - - - -CatalogMap seed - - - -shared insert tail (batcher + inserters) - - - -on-disk artifact - - - -shadow Postgres (post-handoff) - - -edge colour - -━━ - -replication protocol (BASE_BACKUP, START_REPLICATION) - -┄┄ - -walsender wire (post-handoff) - -━━ - -libpq catalog query / CatalogMap lookup - -┄┄ - -CH Native (bootstrap, via shared tail) - -··· - -manifest durability / ack events - -┄┄ - -filesystem (data_dir land, shadow boot) - - -file routing (MultiplexSink) - -catalog filenode -(fn < 16384 ∪ whitelist) - -rail A only - -global/, pg_xact/, pg_control, -tablespace_map, conf files - -rail A only - -pg_replslot/, pg_stat_tmp/, -pg_logical/, pgsql_tmp/, temp_* - -Skip (denylist; -dir entry kept as empty dir) - -user heap -base/<db>/<fn ≥ 16384> - -rail B only -(never lands on shadow) - -pg_toast_<relid> - -rail B decodes on-page TIDs; -mirror put precedes deferred resolution - - - - -seed - -use backup end as -new restart point - - - -out->seed - - - - - -feed - -SourceFeed open -START_REPLICATION -PHYSICAL <end_lsn> - - - -seed->feed - - - - - -manifest - - - -manifest.toml -saved restart state - - - -feed->manifest - - -save on first -status update - - - -feed->shd - - -walsender wire -(steady-state) - - + + Greenfield bootstrap data paths + Physical backup is multiplexed to DiskLanderSink and PageWalkSink. PendingGate resolves row visibility before page rows enter a bootstrap insert tail at backup-start LSN. A concurrent WAL window has its own replay and insert tail at real commit LSNs. Finish both tails and start shadow through backup end before handing off streaming at end_lsn. + + + + + + + + + Source PostgreSQL + BASE_BACKUP and concurrent physical WAL connection + + BackupSource → MultiplexSink + Catalog seed selects files and relations + + SourceFeed → WalReplaySink + Backup-window WAL + Live stream or hydrated archive + + BASE_BACKUP + + WAL + + DiskLanderSink + Catalog + recovery files + + PageWalkSink + Heap pages → tuples + + + + + + shadow_data_dir + Catalog-only backup + + + PendingGate + Resolve tuple visibility + pg_xact + WAL outcomes + + + pg_xact + + PgXactPatch + + Window OwnedTail + Batcher → inserters → ClickHouse + _lsn = real commit position + + committed rows + + Shadow PostgreSQL + Replay through end_lsn + + Bootstrap OwnedTail + ClickHouse INSERT pool + _lsn = backup start + + + + Bootstrap handoff + Finish page + window INSERTs; shadow reaches backup end; start WAL pipeline at end_lsn + + + + window tail drained diff --git a/architecture/catalog.svg b/architecture/catalog.svg new file mode 100644 index 00000000..0b907c70 --- /dev/null +++ b/architecture/catalog.svg @@ -0,0 +1,60 @@ + + Catalog capture and DDL ordering + At a catalog boundary the pump holds later publication, allows shadow replay through that boundary, scans pinned descriptors, persists history and attaches schema events before forwarding the commit. ReorderSink waits for decode placement, flushes batches, waits for durable inserts, then applies DDL or TRUNCATE on its separate ClickHouse connection. + + + + + + + + Capture (log miss) + BoundaryHoldSink + + Shadow PG + replay + worker + + Local state + DescriptorLog / XactBuffer + + Row pipeline + QueueingRecordSink + + + + Publish through boundary L + + Replay reaches L + + SCAN at pinned L + + RelDescriptor batch + + Persist descriptors; attach SchemaEvent(xid) + + Forward commit record; release later WAL + Hold later WAL publication + + DDL / TRUNCATE barrier in ReorderSink + ReorderSink + + Decode / batcher / ack + + ClickHouse + + + Wait until earlier seqs are Placed + + FlushAll, seal earlier rows + + Wait until earlier INSERTs are Acked + + DdlApplicator: CREATE / ALTER / DROP / TRUNCATE + + DDL completes; resume later data + diff --git a/architecture/decoder.dot b/architecture/decoder.dot deleted file mode 100644 index a146a910..00000000 --- a/architecture/decoder.dot +++ /dev/null @@ -1,149 +0,0 @@ -// walshadow — heap-tuple decoder dispatch + type matrix -// One record → DecodedHeap path. Entry through BufferingDecoderSink, -// TRUNCATE intercept, decode_heap_record op nibble dispatch, per-column -// walk, Tier 1/2/3 + PgPending type fan-out. Side branches: replica -// identity old-tuple shape, read-time missing-value substitution. -// -// regeneration spec: -// sources of truth: plans/decoder.md · src/heap_decoder.rs · src/codecs.rs · src/main_data.rs -// subsumes: plans/decoder.md § "Entry point" + "HeapOp variants" + tier overview (detailed tier matrix kept in prose) -// quality bar: -// - 5+ tier outputs all reach `out` without crossing -// - truncate intercept clearly distinct (pre-dispatch) -// - replid annotation hangs off payload without deforming dispatch column -// shared style: palette.md -digraph decoder { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow heap decoder — record → DecodedHeap dispatch + type matrix", fontsize=14, splines=spline, nodesep=0.4, ranksep=0.5, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ ingress ════════ - record [label="WAL Record\nRM_SMGR / RM_HEAP / RM_HEAP2\nrfn = blocks[0].location.rel\nxid, source_lsn, info", fillcolor="#3D3D54", shape=parallelogram]; - - // ════════ sink + truncate intercept ════════ - subgraph cluster_sink { - label="BufferingDecoderSink::on_record (xact_buffer.rs)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - sink [label="on_record\nSMGR CREATE?\nHeap TRUNCATE?\nother heap op?", fillcolor="#4D4128"]; - marker [label="note_smgr_create\nmain-fork generation marker\nproves stash completeness", fillcolor="#4D4128"]; - trunc [label="handle_truncate\nparse_xl_heap_truncate\nfor relid in relids:\n descriptor_by_oid_at_spanned\n → fan out HeapOp::Truncate\n(kind in {'r','p'} only)", fillcolor="#4D4128"]; - relat [label="DescriptorLog::\ndescriptor_at_spanned(rfn, lsn)\nwait-free interval lookup", fillcolor="#4D4D28"]; - stash [label="raw stash\nSpillEntry::Raw\nmain data + block data/images + xid\ncommit: resolve_stash at next_lsn\ntoast | ordinary decode | discard\nAmbiguous = fatal", fillcolor="#4D4128"]; - sink -> marker [label="SMGR CREATE", style=dashed]; - sink -> trunc [label="0x30 TRUNCATE", style=dashed]; - sink -> relat [label="other heap ops"]; - sink -> stash [label="dirty family\n(defer_catalog_decode)", style=dashed]; - relat -> stash [label="NotCovered / Dropped /\nAmbiguous, or marker\ncandidate", style=dashed]; - } - record -> sink; - - // ════════ entry point ════════ - entry [label="decode_heap_record\n(heap_decoder.rs)\ndispatch on rm + info & 0x70", fillcolor="#4D4128"]; - relat -> entry [label="RelDescriptor"]; - - // ════════ op dispatch ════════ - subgraph cluster_ops { - label="op dispatch — info & XLOG_HEAP_OPMASK"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - ins [label="INSERT 0x00\ndecode_insert\nblock.data:\n xl_heap_header (5)\n + bitmap + col data", fillcolor="#4D4128"]; - upd [label="UPDATE 0x20 / HOT 0x40\ndecode_update\nprefix/suffix u16 from\nblock.data per\nXLH_UPDATE_*_FROM_OLD", fillcolor="#4D4128"]; - del [label="DELETE 0x10\ndecode_delete\nold tuple in main_data\nwhen XLH_DELETE_CONTAINS_*", fillcolor="#4D4128"]; - multi [label="MULTI_INSERT 0x50 (Heap2)\ndecode_multi_insert\nntuples loop —\nper-tuple xl_multi_insert_tuple\nsynthesises stripped 5B header", fillcolor="#4D4128"]; - skip [label="LOCK / INPLACE / CONFIRM /\nother RM_HEAP2 ops\n→ empty SmallVec (silent skip)", fillcolor="#4D3A28", shape=note]; - entry -> ins; - entry -> upd; - entry -> del; - entry -> multi; - entry -> skip [style=dashed]; - } - - // ════════ payload walker ════════ - payload [label="decode_tuple_payload\nparse xl_heap_header\nt_infomask2 / t_infomask / t_hoff\nnatts = t_infomask2 & 0x07FF\nbitmap[+MAXALIGN pad]\ncol_data_off = 5 + (t_hoff − 23)", fillcolor="#4D4128"]; - ins -> payload; - upd -> payload [label="cursor +=\nprefix/suffix"]; - del -> payload; - multi -> payload [label="synth header\n+ tuple body"]; - - // ════════ per-column walk ════════ - colwalk [label="per-column walk over rel.attributes\natt_align_nominal (peek byte for varlena)\nbitmap-clear → Some(Null)\nin-prefix / past-EOF → None, partial = true\nidx ≥ natts → missing_value_for(att)", fillcolor="#4D4128"]; - payload -> colwalk; - - // ════════ type dispatch (cluster) ════════ - subgraph cluster_types { - label="decode_one_value — per-OID dispatch"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - tier1 [label="Tier 1 — fixed-width\nbool / char / int2/4/8 / oid\nfloat4/8 / date / time / timestamp[tz]\ntimetz (12) / uuid (16) / name (64)\ninterval (16, via codecs)", fillcolor="#4D4128"]; - tier2 [label="Tier 2 — varlena (typlen = −1)\ndecode_varlena: 1B short / 4B (un)compressed / on-disk TOAST\nbytea / text / bpchar / varchar / json\ninvalid UTF-8 → Bytea fallback", fillcolor="#4D4128"]; - tier3 [label="Tier 3 — in-tree codecs (codecs.rs)\nnumeric (decode_numeric)\ninet / cidr (decode_inet)\ninterval (decode_interval)\njson (passthrough)", fillcolor="#4D4128"]; - pending [label="PgPending { type_oid, raw }\njsonb / range / arrays /\ntsvector / vendor types →\noracle: ENCODE_NATIVE per sealed batch", fillcolor="#5D3F40"]; - toast [label="ExternalToast(ToastPointer)\nvarattrib_1b_e (tag 18)\nva_rawsize / va_extinfo /\nva_valueid / va_toastrelid", fillcolor="#5D3F40", shape=note]; - tier2 -> toast [style=dashed, label="on-disk ptr"]; - } - colwalk -> tier1 [color="#CBA85E"]; - colwalk -> tier2 [color="#BD8183"]; - colwalk -> tier3 [color="#BF8C5F"]; - colwalk -> pending [color="#A1A9CC", style=dashed]; - - // ════════ output ════════ - decoded [label="DecodedHeap {\n rfn, xid, source_lsn,\n op, new, old\n}\nSmallVec<[_; 1]> stack\n(spills only on multi)", fillcolor="#4D4128", shape=parallelogram]; - tier1 -> decoded; - tier2 -> decoded; - tier3 -> decoded; - pending -> decoded; - trunc -> decoded [label="Truncate\n(per relid)", style=dashed]; - - xactbuf [label="XactBuffer::on_heap\nper-xid bucket\n+ TOAST reassembly\n+ subxact tracker", fillcolor="#4D4128", shape=cylinder]; - decoded -> xactbuf [color="#BF8C5F", penwidth=2]; - marker -> xactbuf [style=dashed, color="#CBA85E", label="marker + stash rfn"]; - stash -> xactbuf [color="#BF8C5F", style=dashed, label="Raw"]; - - toastop [label="relkind = 't'\nINSERT → ToastChunk + TID\nDELETE → ToastDelete + TID\nrecord LSN versions mirror row", fillcolor="#4D4128", shape=note]; - decoded -> toastop [style=dashed, color="#BF8C5F"]; - toastop -> xactbuf [style=dashed, color="#BF8C5F"]; - - // ════════ side cluster: replica identity ════════ - subgraph cluster_replident { - label="old-tuple shape — relreplident × flags"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - rid [label=< - - - - - - - -
relreplidentold payload
Default + PKPK cols on OLD_KEY, non-PK = Some(Null) via bitmap
Default no-PKold = None
Nothingold = None
Fullevery non-dropped column
UsingIndexindexed cols, others = Some(Null)
- >, shape=plaintext]; - } - payload -> rid [style=dashed, color="#b380b0", constraint=false, label="UPDATE/DELETE\nold-tuple shape"]; - - // ════════ side cluster: read-time defaults ════════ - subgraph cluster_missing { - label="missing_value_for(att) — fast-path ADD COLUMN ... DEFAULT k"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - miss [label="RelAttr.missing_text = Some(t)\nTier 1 numeric → parse iN / fN\nBOOLOID → PG truth set\nCHAROID → first byte i8\nTEXT/VARCHAR/BPCHAR/NAME/JSON → passthrough\nelse → PgPending { oid, t.as_bytes() }", fillcolor="#5D3F40"]; - } - colwalk -> miss [style=dashed, color="#b380b0", constraint=false, label="idx ≥ natts\n(post-ALTER\ntrailing attrs)"]; - - // ════════ Legend ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - -
node fill — role
WAL record ingress
decoder + xact buffer (heap_decoder.rs, xact_buffer.rs)
ShadowCatalog (RelDescriptor resolution)
silent-skip op (no DecodedHeap emitted)
oracle / missing-value bridge
edge colour — type tier
━━Tier 1 fixed-width (typlen > 0)
━━Tier 2 varlena (typlen = −1) — varatt headers
━━Tier 3 in-tree codecs (numeric / inet / interval / json)
┄┄PgPending → oracle (jsonb / range / array / vendor)
┄┄side branch (replica identity, read-time defaults)
tier matrix — handling
Tier 1inline byte read from buf[abs..abs+typlen]
Tier 2decode_varlena (short / 4B / TOAST ptr) + per-OID body
Tier 3codecs.rs (numeric base-10000, inet family+bits, interval 16B)
PgPendingcarry raw bytes, resolve at emit via shadow PG extension
- >]; - xactbuf -> legend [style=invis]; -} diff --git a/architecture/decoder.svg b/architecture/decoder.svg deleted file mode 100644 index f502e5ba..00000000 --- a/architecture/decoder.svg +++ /dev/null @@ -1,594 +0,0 @@ - - - - - - -decoder - -walshadow heap decoder — record → DecodedHeap dispatch + type matrix - -cluster_sink - -BufferingDecoderSink::on_record (xact_buffer.rs) - - -cluster_ops - -op dispatch — info & XLOG_HEAP_OPMASK - - -cluster_types - -decode_one_value — per-OID dispatch - - -cluster_replident - -old-tuple shape — relreplident × flags - - -cluster_missing - -missing_value_for(att) — fast-path ADD COLUMN ... DEFAULT k - - - -record - -WAL Record -RM_SMGR / RM_HEAP / RM_HEAP2 -rfn = blocks[0].location.rel -xid, source_lsn, info - - - -sink - -on_record -SMGR CREATE? -Heap TRUNCATE? -other heap op? - - - -record->sink - - - - - -marker - -note_smgr_create -main-fork generation marker -proves stash completeness - - - -sink->marker - - -SMGR CREATE - - - -trunc - -handle_truncate -parse_xl_heap_truncate -for relid in relids: -  descriptor_by_oid_at_spanned -  → fan out HeapOp::Truncate -(kind in {'r','p'} only) - - - -sink->trunc - - -0x30 TRUNCATE - - - -relat - -DescriptorLog:: -descriptor_at_spanned(rfn, lsn) -wait-free interval lookup - - - -sink->relat - - -other heap ops - - - -stash - -raw stash -SpillEntry::Raw -main data + block data/images + xid -commit: resolve_stash at next_lsn -toast | ordinary decode | discard -Ambiguous = fatal - - - -sink->stash - - -dirty family -(defer_catalog_decode) - - - -xactbuf - - -XactBuffer::on_heap -per-xid bucket -+ TOAST reassembly -+ subxact tracker - - - -marker->xactbuf - - -marker + stash rfn - - - -decoded - -DecodedHeap { -  rfn, xid, source_lsn, -  op, new, old -} -SmallVec<[_; 1]> stack -(spills only on multi) - - - -trunc->decoded - - -Truncate -(per relid) - - - -relat->stash - - -NotCovered / Dropped / -Ambiguous, or marker -candidate - - - -entry - -decode_heap_record -(heap_decoder.rs) -dispatch on rm + info & 0x70 - - - -relat->entry - - -RelDescriptor - - - -stash->xactbuf - - -Raw - - - -ins - -INSERT 0x00 -decode_insert -block.data: -  xl_heap_header (5) -  + bitmap + col data - - - -entry->ins - - - - - -upd - -UPDATE 0x20 / HOT 0x40 -decode_update -prefix/suffix u16 from -block.data per -XLH_UPDATE_*_FROM_OLD - - - -entry->upd - - - - - -del - -DELETE 0x10 -decode_delete -old tuple in main_data -when XLH_DELETE_CONTAINS_* - - - -entry->del - - - - - -multi - -MULTI_INSERT 0x50 (Heap2) -decode_multi_insert -ntuples loop — -per-tuple xl_multi_insert_tuple -synthesises stripped 5B header - - - -entry->multi - - - - - -skip - - - -LOCK / INPLACE / CONFIRM / -other RM_HEAP2 ops -→ empty SmallVec (silent skip) - - - -entry->skip - - - - - -payload - -decode_tuple_payload -parse xl_heap_header -t_infomask2 / t_infomask / t_hoff -natts = t_infomask2 & 0x07FF -bitmap[+MAXALIGN pad] -col_data_off = 5 + (t_hoff − 23) - - - -ins->payload - - - - - -upd->payload - - -cursor += -prefix/suffix - - - -del->payload - - - - - -multi->payload - - -synth header -+ tuple body - - - -colwalk - -per-column walk over rel.attributes -att_align_nominal (peek byte for varlena) -bitmap-clear → Some(Null) -in-prefix / past-EOF → None, partial = true -idx ≥ natts → missing_value_for(att) - - - -payload->colwalk - - - - - -rid - - - -relreplident - - -old payload - -Default + PK - -PK cols on OLD_KEY, non-PK = Some(Null) via bitmap - -Default no-PK - -old = None - -Nothing - -old = None - -Full - -every non-dropped column - -UsingIndex - -indexed cols, others = Some(Null) - - - -payload->rid - - -UPDATE/DELETE -old-tuple shape - - - -tier1 - -Tier 1 — fixed-width -bool / char / int2/4/8 / oid -float4/8 / date / time / timestamp[tz] -timetz (12) / uuid (16) / name (64) -interval (16, via codecs) - - - -colwalk->tier1 - - - - - -tier2 - -Tier 2 — varlena (typlen = −1) -decode_varlena: 1B short / 4B (un)compressed / on-disk TOAST -bytea / text / bpchar / varchar / json -invalid UTF-8 → Bytea fallback - - - -colwalk->tier2 - - - - - -tier3 - -Tier 3 — in-tree codecs (codecs.rs) -numeric (decode_numeric) -inet / cidr (decode_inet) -interval (decode_interval) -json (passthrough) - - - -colwalk->tier3 - - - - - -pending - -PgPending { type_oid, raw } -jsonb / range / arrays / -tsvector / vendor types → -oracle: ENCODE_NATIVE per sealed batch - - - -colwalk->pending - - - - - -miss - -RelAttr.missing_text = Some(t) -Tier 1 numeric → parse iN / fN -BOOLOID → PG truth set -CHAROID → first byte i8 -TEXT/VARCHAR/BPCHAR/NAME/JSON → passthrough -else → PgPending { oid, t.as_bytes() } - - - -colwalk->miss - - -idx ≥ natts -(post-ALTER -trailing attrs) - - - -tier1->decoded - - - - - -toast - - - -ExternalToast(ToastPointer) -varattrib_1b_e (tag 18) -va_rawsize / va_extinfo / -va_valueid / va_toastrelid - - - -tier2->toast - - -on-disk ptr - - - -tier2->decoded - - - - - -tier3->decoded - - - - - -pending->decoded - - - - - -decoded->xactbuf - - - - - -toastop - - - -relkind = 't' -INSERT → ToastChunk + TID -DELETE → ToastDelete + TID -record LSN versions mirror row - - - -decoded->toastop - - - - - -legend - - - -node fill — role - - - -WAL record ingress - - - -decoder + xact buffer (heap_decoder.rs, xact_buffer.rs) - - - -ShadowCatalog (RelDescriptor resolution) - - - -silent-skip op (no DecodedHeap emitted) - - - -oracle / missing-value bridge - - -edge colour — type tier - -━━ - -Tier 1 fixed-width (typlen > 0) - -━━ - -Tier 2 varlena (typlen = −1) — varatt headers - -━━ - -Tier 3 in-tree codecs (numeric / inet / interval / json) - -┄┄ - -PgPending → oracle (jsonb / range / array / vendor) - -┄┄ - -side branch (replica identity, read-time defaults) - - -tier matrix — handling - -Tier 1 - -inline byte read from buf[abs..abs+typlen] - -Tier 2 - -decode_varlena (short / 4B / TOAST ptr) + per-OID body - -Tier 3 - -codecs.rs (numeric base-10000, inet family+bits, interval 16B) - -PgPending - -carry raw bytes, resolve at emit via shadow PG extension - - - - -toastop->xactbuf - - - - - diff --git a/architecture/emitter.dot b/architecture/emitter.dot deleted file mode 100644 index ccf5d76d..00000000 --- a/architecture/emitter.dot +++ /dev/null @@ -1,145 +0,0 @@ -// walshadow — CH emitter component view -// Parallel decode+insert pipeline: reorder coordinator (commit order, -// side-effect-free transaction plan then execute, DDL/TRUNCATE barrier) -// → decode pool ×M → InsertBatcher (per-table -// TableEncoder, budget/deadline seal) → inserter pool ×N (one complete -// INSERT per sealed batch) → ack collector (contiguous-done watermark). -// Zoomed-in view of cluster_ch + cluster_ddl from internals.dot. -// -// regeneration spec: -// sources of truth: plans/emitter.md · src/emit/pipeline/{reorder,planner,plan_spool,decode,batcher,inserter,ack,tail}.rs · src/emit/{ch_emitter,ch_ddl}.rs · src/catalog/type_bridge.rs -// subsumes: plans/emitter.md § "Stage walk" + "Transaction planner" + "Barrier fence" + "Ack-LSN tracking" + "DdlApplicator" -// quality bar: -// - decode ×M and inserter ×N read as pools (stacked node or ×M/×N label), not single tasks -// - barrier fence visually orders DDL strictly after earlier data durable (placed → FlushAll → durable) -// - ack side-channel (Register/Placed/Acked → watermark) distinct from row path; dotted #b380b0 -// - rows path solid #BF8C5F, DDL path dashed #BF8C5F — same colour, different style -// shared style: palette.md -digraph emitter { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow emitter — one ordered pipeline, ClickHouse or metrics only", fontsize=14, splines=spline, nodesep=0.4, ranksep=0.5, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ═════ reorder coordinator (upstream, commit-order boundary) ═════ - subgraph cluster_src { - label="reorder coordinator — single-threaded commit order (emit/pipeline/reorder.rs, inner sink of QueueingRecordSink)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - reorder [label="transaction reorder\nwalk rows and changes in WAL order\nempty commits and aborts still advance progress\nschema and truncate changes wait for earlier rows", fillcolor="#4D4128"]; - plan [label="transaction planner (planner.rs + plan_spool.rs)\nplan first, execute after: detoast + route +\nraw decode, side-effect-free; frozen route view\n(whole-xact config granularity)\nplan ≤1 MiB resident else .plan spool\n[len|crc32c|body]*, seal frame; error = abandon,\nnothing emitted", fillcolor="#4D4128"]; - cat [label="ShadowCatalog::subscribe\nSchemaEvent\nAdded / Changed / Dropped\n(unbounded mpsc, rides xact buffer\n as ordered_events)", fillcolor="#4D4D28", shape=parallelogram]; - reorder -> plan [label="drain walk\n(plan → seal → execute)"]; - } - - // ═════ decode pool ═════ - subgraph cluster_decode { - label="decode pool ×M (emit/pipeline/decode.rs)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - decode [label="decode worker ×M\nplanned envelopes: descriptor + route attached,\nvalues resolved at planning\noracle columns stay raw for the inserter\nchunk 1024 rows / 4 MiB", fillcolor="#4D4128"]; - } - - // ═════ batcher hub ═════ - subgraph cluster_batch { - label="InsertBatcher — single hub, per-table accumulation (emit/pipeline/batcher.rs)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - enc [label="TableEncoder per dest table\nColumnBuf slabs: Fixed / String /\nNullableFixed / NullableString\ncolumn-major + synthetic\n_lsn / _xid / _commit_ts / _is_deleted", fillcolor="#5D4628", shape=note]; - trip [label="flush trigger\nrow_budget 65536 · byte_budget 1 MiB\nper-table deadline (flush_timeout,\n 0 → 100 ms floor) · FlushAll", fillcolor="#5D4628", shape=diamond]; - seal [label="seal InsertBatch\nowned slabs + per_seq row counts\nFlushAll also bumps schema_epoch\n(rebuild plans post-DDL)", fillcolor="#5D4628"]; - - enc -> trip -> seal; - } - - // ═════ inserter pool ═════ - subgraph cluster_insert { - label="inserter pool ×N (emit/pipeline/inserter.rs)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - ins [label="inserter ×N — any idle takes any batch\nTypeAst cache per (table, epoch)\noracle: one request per sealed batch,\nsplice returned Native columns\nBlockBuilder over owned slabs\nsend_query → send_data → EndOfStream\nsend_with_retry: reconnect + backoff,\ninsert_timeout 30 s; exhaustion = Fatal", fillcolor="#5D4628"]; - } - - // ═════ DDL path: barrier + applicator (own CH connection) ═════ - subgraph cluster_ddl { - label="ordered control barriers — DDL / config / TOAST lifecycle"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - fence [label="barrier_fence\n1. wait all seqs placed\n2. batcher FlushAll (+ reply)\n3. wait all seqs durable", fillcolor="#5D4628"]; - bridge [label="type_bridge::map\nRelAttr → ResolvedColumn\n(reject type change,\n pk strips Nullable)", fillcolor="#5D4628"]; - apply [label="apply ordered control\nCatalog → DdlApplicator\nConfig → resolver republish\nowner TRUNCATE → dest + toast mirror wipe\nToastBarrier → rewrite O−B\ntoast Dropped → durable retire enqueue", fillcolor="#5D4628"]; - - fence -> apply; - bridge -> apply [style=dashed, label="resolve types"]; - } - - // ═════ CH endpoint ═════ - subgraph cluster_ch { - label="ClickHouse — main insert pool + DDL + TOAST store connections"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - chrows [label="insert connections ×N\nclickhouse-c-rs AsyncClient\nReplacingMergeTree(_lsn, _is_deleted)\nNative rows", fillcolor="#4D4128", shape=cylinder]; - chddl [label="DDL connection\nclickhouse-c-rs AsyncClient\nALTER / CREATE /\n DROP / TRUNCATE", fillcolor="#4D4128", shape=cylinder]; - chtoast [label="TOAST store connection\npg_toast_ mirrors\nput / as-of fetch / truncate / O−B", fillcolor="#4D4128", shape=cylinder]; - } - - toastput [label="ToastResolver\nput births + tombstones before publish\nas-of fetch for pre-window values", fillcolor="#4D4D28"]; - ledger [label="toast_retires.toml\nsaved mirror retirements", fillcolor="#4D3850", shape=note]; - - nulltail [label="metrics-only tail\nno ClickHouse connection\nacknowledge rows immediately", fillcolor="#4D3A28", shape=note]; - - // ═════ ack collector (durability watermark) ═════ - ack [label="ack collector\nwait for every earlier commit\npublish latest durable position\nfeed manifest + source feedback", fillcolor="#4D3A28", shape=note]; - - // ═════ row path ═════ - plan -> decode [color="#BF8C5F", penwidth=2, lhead=cluster_decode, label="execute_plan:\nDecodeJob\n(mpmc, bound 4M)"]; - decode -> enc [color="#BF8C5F", penwidth=2, lhead=cluster_batch, label="BatcherMsg::Rows\n(FIFO mpsc 256 —\nshared with FlushAll)"]; - seal -> ins [color="#BF8C5F", penwidth=2, lhead=cluster_insert, label="InsertBatch\n(mpmc)"]; - ins -> chrows [color="#BF8C5F", penwidth=2, label="one complete INSERT\nper sealed batch"]; - decode -> nulltail [color="#BF8C5F", style=dashed, constraint=false, label="metrics only"]; - reorder -> toastput [color="#BF8C5F", style=dashed, label="TOAST changes"]; - decode -> toastput [color="#CBA85E", style=dashed, label="fetch"]; - toastput -> chtoast [color="#BF8C5F", penwidth=2]; - - // ═════ DDL path ═════ - cat -> reorder [color="#CBA85E", style=dashed, label="SchemaEvent\ndrains at commit"]; - reorder -> fence [color="#BF8C5F", style=dashed, lhead=cluster_ddl, label="barrier xact:\nper event / TRUNCATE"]; - apply -> chddl [color="#BF8C5F", style=dashed, penwidth=2, label="DDL SQL\n(own connection)"]; - apply -> chtoast [color="#BF8C5F", style=dashed, label="truncate / O−B"]; - apply -> ledger [color="#6E6963", style=dashed, label="toast DROP enqueue"]; - ledger -> chtoast [color="#BF8C5F", style=dashed, label="retire after saved\nrestart point passes"]; - fence -> trip [style=dashed, color="#B58B86", constraint=false, label="FlushAll"]; - - // ═════ ack side-channel ═════ - reorder -> ack [style=dotted, color="#b380b0", constraint=false, label="Register(seq, commit_lsn)"]; - decode -> ack [style=dotted, color="#b380b0", constraint=false, label="Placed(seq, rows)"]; - ins -> ack [style=dotted, color="#b380b0", label="Acked(per_seq)\nonly after EndOfStream"]; - nulltail -> ack [style=dotted, color="#b380b0", constraint=false, label="done"]; - ack -> fence [style=dotted, color="#b380b0", constraint=false, label="wait_placed_through /\nwait_through"]; - - // ═════ Legend ═════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - -
node fill — role
reorder / decode pool / ClickHouse
ShadowCatalog schema-event tx
batcher / inserters / DdlApplicator
ack collector / null tail / manifest side-channel
edge style
━━row path (Native rows, solid)
┄┄DDL path (own connection, dashed)
┄┄SchemaEvent off ShadowCatalog
···ack events / barrier waits
synthetic columns (every dest table)
_lsnUInt64 — commit_lsn; ReplacingMergeTree dedup key
_xidUInt32 — source xid; recovers xact boundary
_commit_tsDateTime64(6, 'UTC'); shifted from PG epoch
_is_deletedBool — 1 on delete; ReplacingMergeTree is_deleted arg unless soft_delete
watermark rule
commits may finish out of order; progress advances only after every earlier commit is durable. Failed batch stops process so restart can replay it.
- >]; - chrows -> legend [style=invis]; - chddl -> legend [style=invis]; -} diff --git a/architecture/emitter.svg b/architecture/emitter.svg deleted file mode 100644 index 32f1d234..00000000 --- a/architecture/emitter.svg +++ /dev/null @@ -1,477 +0,0 @@ - - - - - - -emitter - -walshadow emitter — one ordered pipeline, ClickHouse or metrics only - -cluster_src - -reorder coordinator — single-threaded commit order  (emit/pipeline/reorder.rs, inner sink of QueueingRecordSink) - - -cluster_decode - -decode pool ×M  (emit/pipeline/decode.rs) - - -cluster_batch - -InsertBatcher — single hub, per-table accumulation  (emit/pipeline/batcher.rs) - - -cluster_insert - -inserter pool ×N  (emit/pipeline/inserter.rs) - - -cluster_ddl - -ordered control barriers — DDL / config / TOAST lifecycle - - -cluster_ch - -ClickHouse — main insert pool + DDL + TOAST store connections - - - -reorder - -transaction reorder -walk rows and changes in WAL order -empty commits and aborts still advance progress -schema and truncate changes wait for earlier rows - - - -plan - -transaction planner (planner.rs + plan_spool.rs) -plan first, execute after: detoast + route + -raw decode, side-effect-free; frozen route view -(whole-xact config granularity) -plan ≤1 MiB resident else .plan spool -[len|crc32c|body]*, seal frame; error = abandon, -nothing emitted - - - -reorder->plan - - -drain walk -(plan → seal → execute) - - - -fence - -barrier_fence -1. wait all seqs placed -2. batcher FlushAll (+ reply) -3. wait all seqs durable - - - -reorder->fence - - -barrier xact: -per event / TRUNCATE - - - -toastput - -ToastResolver -put births + tombstones before publish -as-of fetch for pre-window values - - - -reorder->toastput - - -TOAST changes - - - -ack - - - -ack collector -wait for every earlier commit -publish latest durable position -feed manifest + source feedback - - - -reorder->ack - - -Register(seq, commit_lsn) - - - -decode - -decode worker ×M -planned envelopes: descriptor + route attached, -values resolved at planning -oracle columns stay raw for the inserter -chunk 1024 rows / 4 MiB - - - -plan->decode - - -execute_plan: -DecodeJob -(mpmc, bound 4M) - - - -cat - -ShadowCatalog::subscribe -SchemaEvent -Added / Changed / Dropped -(unbounded mpsc, rides xact buffer - as ordered_events) - - - -cat->reorder - - -SchemaEvent -drains at commit - - - -enc - - - -TableEncoder per dest table -ColumnBuf slabs: Fixed / String / -NullableFixed / NullableString -column-major + synthetic -_lsn / _xid / _commit_ts / _is_deleted - - - -decode->enc - - -BatcherMsg::Rows -(FIFO mpsc 256 — -shared with FlushAll) - - - -decode->toastput - - -fetch - - - -nulltail - - - -metrics-only tail -no ClickHouse connection -acknowledge rows immediately - - - -decode->nulltail - - -metrics only - - - -decode->ack - - -Placed(seq, rows) - - - -trip - -flush trigger -row_budget 65536 · byte_budget 1 MiB -per-table deadline (flush_timeout, - 0 → 100 ms floor) · FlushAll - - - -enc->trip - - - - - -seal - -seal InsertBatch -owned slabs + per_seq row counts -FlushAll also bumps schema_epoch -(rebuild plans post-DDL) - - - -trip->seal - - - - - -ins - -inserter ×N — any idle takes any batch -TypeAst cache per (table, epoch) -oracle: one request per sealed batch, -splice returned Native columns -BlockBuilder over owned slabs -send_query → send_data → EndOfStream -send_with_retry: reconnect + backoff, -insert_timeout 30 s; exhaustion = Fatal - - - -seal->ins - - -InsertBatch -(mpmc) - - - -chrows - - -insert connections ×N -clickhouse-c-rs AsyncClient -ReplacingMergeTree(_lsn, _is_deleted) -Native rows - - - -ins->chrows - - -one complete INSERT -per sealed batch - - - -ins->ack - - -Acked(per_seq) -only after EndOfStream - - - -fence->trip - - -FlushAll - - - -apply - -apply ordered control -Catalog → DdlApplicator -Config → resolver republish -owner TRUNCATE → dest + toast mirror wipe -ToastBarrier → rewrite O−B -toast Dropped → durable retire enqueue - - - -fence->apply - - - - - -bridge - -type_bridge::map -RelAttr → ResolvedColumn -(reject type change, - pk strips Nullable) - - - -bridge->apply - - -resolve types - - - -chddl - - -DDL connection -clickhouse-c-rs AsyncClient -ALTER / CREATE / - DROP / TRUNCATE - - - -apply->chddl - - -DDL SQL -(own connection) - - - -chtoast - - -TOAST store connection -pg_toast_<relid> mirrors -put / as-of fetch / truncate / O−B - - - -apply->chtoast - - -truncate / O−B - - - -ledger - - - -toast_retires.toml -saved mirror retirements - - - -apply->ledger - - -toast DROP enqueue - - - -legend - - - -node fill — role - - - -reorder / decode pool / ClickHouse - - - -ShadowCatalog schema-event tx - - - -batcher / inserters / DdlApplicator - - - -ack collector / null tail / manifest side-channel - - -edge style - -━━ - -row path (Native rows, solid) - -┄┄ - -DDL path (own connection, dashed) - -┄┄ - -SchemaEvent off ShadowCatalog - -··· - -ack events / barrier waits - - -synthetic columns (every dest table) - -_lsn - -UInt64 — commit_lsn; ReplacingMergeTree dedup key - -_xid - -UInt32 — source xid; recovers xact boundary - -_commit_ts - -DateTime64(6, 'UTC'); shifted from PG epoch - -_is_deleted - -Bool — 1 on delete; ReplacingMergeTree is_deleted arg unless soft_delete - - -watermark rule - -commits may finish out of order; progress advances only after every earlier commit is durable. Failed batch stops process so restart can replay it. - - - - - -toastput->chtoast - - - - - -ledger->chtoast - - -retire after saved -restart point passes - - - -nulltail->ack - - -done - - - -ack->fence - - -wait_placed_through / -wait_through - - - diff --git a/architecture/filter.dot b/architecture/filter.dot deleted file mode 100644 index 58f57dbb..00000000 --- a/architecture/filter.dot +++ /dev/null @@ -1,132 +0,0 @@ -// walshadow — per-record filter decision pipeline -// StreamingWalker → Filter::decide → keep verbatim or NOOP-rewrite + -// CRC32C → segment buffer. CatalogTracker side cluster feeds the -// decision (relmap + pg_class harvest + seed). rmgr keep policy is a -// decision sub-table off to the side. -// -// regeneration spec: -// sources of truth: plans/filter.md · src/filter/{engine,dirty_tree}.rs · src/filter/catalog_tracker.rs · src/filter/rewrite.rs -// subsumes: plans/filter.md § "Filter contract" + "Dirty tree" + "Rewrite over fork" -// quality bar: -// - track cluster doesn't push decide off main column -// - noop / rewrite siblings rank-aligned so segbuf joins cleanly -// - rmgr keep-policy table in legend fits within graph width -// shared style: palette.md -digraph filter { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow filter — per-record keep/drop + byte-preserving NOOP rewrite", fontsize=14, splines=spline, nodesep=0.45, ranksep=0.65, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ ingress ════════ - subgraph cluster_in { - label="① ingress"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - chunk [label="WalChunk\n(source-order bytes,\npage-framed)", fillcolor="#3D3D54", shape=parallelogram]; - walker [label="StreamingWalker\nstitch records across pages,\nyield (logical_bytes,\n byte_ranges, page_magic)", fillcolor="#3D3D54"]; - parse [label="parse_record_from_bytes\nwal-rus XLogRecord\nblocks + main_data", fillcolor="#3D3D54"]; - chunk -> walker -> parse; - } - - // ════════ filter::decide (main column) ════════ - subgraph cluster_decide { - label="② Filter::decide (per record, post-update tracker)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - observe [label="tracker.observe(rec)\nupdate catalog set\nbefore classify", fillcolor="#4D3A28"]; - classify [label="classify::classify\nSpecial / Catalog /\nUser / Empty", fillcolor="#4D3A28"]; - empty [label="main_data::relation_for_empty\nXLOG_HEAP2_NEW_CID (0x70)\nXLOG_BTREE_REUSE_PAGE (0xD0)\nextract RelFileLocator", fillcolor="#4D3A28"]; - iscat [label="tracker.is_catalog\n(db_node, rel_node)\n+ shared (0, rel_node)", fillcolor="#4D3A28", shape=diamond]; - - keep [label="Decision::Keep\n(Special, Catalog,\n promoted User,\n Empty rel ∈ cat,\n safe-default Empty)", fillcolor="#4D3A28"]; - drop [label="Decision::Drop\n(User no-cat-ref,\n Empty rel ∉ catalog)", fillcolor="#4D3A28"]; - - observe -> classify; - classify -> iscat [label="User /\nCatalog"]; - classify -> empty [label="Empty\n(no blocks)"]; - empty -> iscat [label="rel from\nmain_data"]; - iscat -> keep [label="yes"]; - iscat -> drop [label="no"]; - {rank=same; keep; drop} - } - parse -> observe [color="#A1A9CC", penwidth=2, lhead=cluster_decide]; - - // ════════ rmgr keep policy — sub-table inside decide cluster, pinned right ════════ - policy [shape=plaintext, label=< - - - - - - - - - -
rmgr keep policy
HEAP / HEAP2keep iff block ref ∈ catalog set
BTREEkeep iff block ref ∈ catalog set
HASH / GIN / GIST / SPGIST / BRINdrop on user index
SEQ / GENERIC / LOGICALMSGdrop when no catalog ref
RELMAP / XACT / CLOG / MULTIXACT / STANDBYalways keep
COMMIT_TS / REPL_ORIGIN / DBASE / TBLSPC / SMGRalways keep
XLOGCHECKPOINT / NEXTOID / PARAMETER_CHANGE keep
- >]; - classify -> policy [style=dashed, color="#6E6963", constraint=false, label="rmgr →\nclass", arrowhead=none]; - - // ════════ CatalogTracker side cluster ════════ - subgraph cluster_track { - label="③ CatalogTracker (live catalog filenode set)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - nodes [label=< - - - - - - - - -
nodes: HashSet<(db, rel)>
bootstrap: rel < 16384 ∨ shared (db=0)
tracker.observe(rec) dispatches:
• RM_RELMAP_ID → mapoid → filenum (magic 0x592717)
• pg_class HEAP insert/update → (oid, relfilenode)
• pg_class heap_delete → coarse epoch++ (DROP signal)
• seed_from_source → libpq SELECT pg_class at attach
- >, fillcolor="#4D4D28", shape=box]; - } - observe -> nodes [style=dashed, color="#CBA85E", label="tracker.observe(rec)"]; - nodes -> iscat [style=dashed, color="#CBA85E", label="is_catalog(db, rel)"]; - - // ════════ dirty tree (catalog-dirty xact families) ════════ - dirty [label="dirty tree (dirty_tree.rs)\ncatalog touch marks writing xid;\nlinks: inline toplevel xid +\nXLOG_XACT_ASSIGNMENT\nsubxact abort drops subtree only\ncommit/abort clears family", fillcolor="#4D4D28"]; - stamp [label="Record.defer_catalog_decode\nevery decoder-routed record of a\ndirty family → raw stash at the\ndecoder sink (see xact diagram)", fillcolor="#4D3850", shape=note]; - observe -> dirty [style=dashed, color="#CBA85E", label="catalog write\n→ mark xid dirty"]; - dirty -> stamp [style=dashed, color="#CBA85E", constraint=false, label="is_dirty(xid)"]; - - // ════════ rewrite + emit ════════ - subgraph cluster_rw { - label="④ rewrite + emit"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - verbatim [label="copy logical_bytes\ninto segment buffer\nat byte_ranges", fillcolor="#4D3A28"]; - noop [label="rewrite::noop_replace\nheader: keep xl_tot_len, xl_prev;\n info=XLOG_NOOP, rmid=RM_XLOG\nbody: zero-fill + main_data marker\n SHORT (≤257) | LONG", fillcolor="#4D3A28"]; - crc [label="CRC32C recompute\nINIT → body → header[0..20]\nmatches xlog.c:5169", fillcolor="#4D3A28"]; - scatter [label="scatter rewritten bytes\ninto segment buffer\nat each byte_range\n(cross-seg: rewrite both)", fillcolor="#4D3A28"]; - {rank=same; verbatim; noop} - noop -> crc -> scatter; - } - keep -> verbatim [color="#BD8183"]; - drop -> noop [color="#BD8183"]; - - // ════════ output ════════ - segbuf [label="16 MiB segment buffer\nfiltered image\nfiltered_lsn == source_lsn", fillcolor="#4D3850", shape=note]; - manifest [label="Manifest\nrecords: [{offset, len,\n rmid, info, kind}]\n+ FilterStats delta", fillcolor="#4D3850", shape=note]; - verbatim -> segbuf; - scatter -> segbuf; - segbuf -> manifest [style=dashed]; - - // ════════ Legend ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - -
node fill — role
ingress (walker, parser)
walshadow filter / rewrite (sync)
CatalogTracker state + inputs
on-disk / output artifact
edge colour
━━source replication frame
━━decision → rewrite (hot path)
┄┄tracker update + catalog lookup
source
src/filter.rs — Filter::decide
src/catalog_tracker.rs — CatalogTracker
src/rewrite.rs — noop_replace + CRC32C
src/main_data.rs — empty reclassifier
- >]; - manifest -> legend [style=invis]; -} diff --git a/architecture/filter.svg b/architecture/filter.svg deleted file mode 100644 index 2ccc2f31..00000000 --- a/architecture/filter.svg +++ /dev/null @@ -1,433 +0,0 @@ - - - - - - -filter - -walshadow filter — per-record keep/drop + byte-preserving NOOP rewrite - -cluster_in - -① ingress - - -cluster_decide - -② Filter::decide  (per record, post-update tracker) - - -cluster_track - -③ CatalogTracker  (live catalog filenode set) - - -cluster_rw - -④ rewrite + emit - - - -chunk - -WalChunk -(source-order bytes, -page-framed) - - - -walker - -StreamingWalker -stitch records across pages, -yield (logical_bytes, - byte_ranges, page_magic) - - - -chunk->walker - - - - - -parse - -parse_record_from_bytes -wal-rus XLogRecord -blocks + main_data - - - -walker->parse - - - - - -observe - -tracker.observe(rec) -update catalog set -before classify - - - -parse->observe - - - - - -classify - -classify::classify -Special / Catalog / -User / Empty - - - -observe->classify - - - - - -nodes - -nodes -: HashSet<(db, rel)> -bootstrap: rel < 16384 ∨ shared (db=0) -tracker.observe(rec) dispatches: -• RM_RELMAP_ID → mapoid → filenum (magic 0x592717) -• pg_class HEAP insert/update → (oid, relfilenode) -• pg_class heap_delete → coarse epoch++ (DROP signal) -• seed_from_source → libpq SELECT pg_class at attach - - - -observe->nodes - - -tracker.observe(rec) - - - -dirty - -dirty tree (dirty_tree.rs) -catalog touch marks writing xid; -links: inline toplevel xid + -XLOG_XACT_ASSIGNMENT -subxact abort drops subtree only -commit/abort clears family - - - -observe->dirty - - -catalog write -→ mark xid dirty - - - -empty - -main_data::relation_for_empty -XLOG_HEAP2_NEW_CID (0x70) -XLOG_BTREE_REUSE_PAGE (0xD0) -extract RelFileLocator - - - -classify->empty - - -Empty -(no blocks) - - - -iscat - -tracker.is_catalog -(db_node, rel_node) -+ shared (0, rel_node) - - - -classify->iscat - - -User / -Catalog - - - -policy - - - - -rmgr keep policy - -HEAP / HEAP2 - -keep iff block ref ∈ catalog set - -BTREE - -keep iff block ref ∈ catalog set - -HASH / GIN / GIST / SPGIST / BRIN - -drop on user index - -SEQ / GENERIC / LOGICALMSG - -drop when no catalog ref - -RELMAP / XACT / CLOG / MULTIXACT / STANDBY - -always keep - -COMMIT_TS / REPL_ORIGIN / DBASE / TBLSPC / SMGR - -always keep - -XLOG - -CHECKPOINT / NEXTOID / PARAMETER_CHANGE keep - - - -classify->policy - -rmgr → -class - - - -empty->iscat - - -rel from -main_data - - - -keep - -Decision::Keep -(Special, Catalog, - promoted User, - Empty rel ∈ cat, - safe-default Empty) - - - -iscat->keep - - -yes - - - -drop - -Decision::Drop -(User no-cat-ref, - Empty rel ∉ catalog) - - - -iscat->drop - - -no - - - -verbatim - -copy logical_bytes -into segment buffer -at byte_ranges - - - -keep->verbatim - - - - - -noop - -rewrite::noop_replace -header: keep xl_tot_len, xl_prev; - info=XLOG_NOOP, rmid=RM_XLOG -body: zero-fill + main_data marker - SHORT (≤257) | LONG - - - -drop->noop - - - - - -nodes->iscat - - -is_catalog(db, rel) - - - -stamp - - - -Record.defer_catalog_decode -every decoder-routed record of a -dirty family → raw stash at the -decoder sink (see xact diagram) - - - -dirty->stamp - - -is_dirty(xid) - - - -segbuf - - - -16 MiB segment buffer -filtered image -filtered_lsn == source_lsn - - - -verbatim->segbuf - - - - - -crc - -CRC32C recompute -INIT → body → header[0..20] -matches xlog.c:5169 - - - -noop->crc - - - - - -scatter - -scatter rewritten bytes -into segment buffer -at each byte_range -(cross-seg: rewrite both) - - - -crc->scatter - - - - - -scatter->segbuf - - - - - -manifest - - - -Manifest -records: [{offset, len, - rmid, info, kind}] -+ FilterStats delta - - - -segbuf->manifest - - - - - -legend - - - -node fill — role - - - -ingress (walker, parser) - - - -walshadow filter / rewrite (sync) - - - -CatalogTracker state + inputs - - - -on-disk / output artifact - - -edge colour - -━━ - -source replication frame - -━━ - -decision → rewrite (hot path) - -┄┄ - -tracker update + catalog lookup - - -source - -src/filter.rs - — Filter::decide - -src/catalog_tracker.rs - — CatalogTracker - -src/rewrite.rs - — noop_replace + CRC32C - -src/main_data.rs - — empty reclassifier - - - - diff --git a/architecture/internals.dot b/architecture/internals.dot deleted file mode 100644 index 3e0839c7..00000000 --- a/architecture/internals.dot +++ /dev/null @@ -1,177 +0,0 @@ -// walshadow — internal pipeline detail -// Deeper than overview.dot: filter stages, fan-out ordering, queueing -// stage, catalog cache internals, walsender server internals, decoder -// type matrix, xact spill, schema-event channel, … -digraph internals { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow internals — pipeline stages, taps & caches", fontsize=14, splines=spline, nodesep=0.45, ranksep=0.55, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ External actors ════════ - src [label="source PG", fillcolor="#3D3D54", shape=cylinder]; - shd [label="shadow PG\nwalreceiver + catalog\n(pg_last_wal_replay_lsn)", fillcolor="#3D4128", shape=cylinder]; - ch [label="ClickHouse", fillcolor="#4D4128", shape=cylinder]; - - // ════════ ① ingress ════════ - subgraph cluster_ingress { - label="① ingress (main loop, tokio)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - feed [label="SourceFeed::pump\nwal-rus ReplicationConn\nIDENTIFY_SYSTEM +\nSTART_REPLICATION PHYSICAL", fillcolor="#3D3D54"]; - chunks [label="WalChunk stream\n(start_lsn, server_wal_end,\n bytes)", fillcolor="#3D3D54", shape=parallelogram]; - feed -> chunks; - } - - // ════════ ② filter pipeline ════════ - subgraph cluster_filter { - label="② filter pipeline (sync, record cadence)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - walker [label="StreamingWalker\npage-state machine\nrecord stitch across pages", fillcolor="#4D3A28"]; - track [label="CatalogTracker\nrelfilenode set\n(relmap + pg_class writes)", fillcolor="#4D3A28"]; - decide [label="Filter::decide\nKeep catalog\nDrop user-heap\nEmpty → reclass", fillcolor="#4D3A28"]; - rewrite [label="rewrite::noop_replace\nin-place at byte_ranges\n+ CRC32C recompute", fillcolor="#4D3A28"]; - segbuf [label="16 MiB segment buffer\n(rewritten image)", fillcolor="#4D3A28", shape=cylinder]; - walker -> decide -> rewrite -> segbuf; - decide -> track [dir=both, arrowtail=open, style=dashed]; - } - chunks -> walker [color="#A1A9CC", penwidth=2]; - - // ════════ ③ fan-out (pump task — order matters) ════════ - subgraph cluster_fanout { - label="③ CompositeRecordSink fan-out (pump task, sync — ordering matters)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - bytesink [label="❶ ShadowStreamSink\non_record_bytes\n(stays on pump)", fillcolor="#4D3340"]; - recsink [label="❷ on_record →\nQueueingRecordSink\n(clones + enqueues,\n returns immediately)", fillcolor="#4D3340"]; - segsink [label="❸ DirSegmentSink\non_segment\n(at 16 MiB boundary)", fillcolor="#4D3340"]; - } - segbuf -> bytesink [color="#BD8183", penwidth=2]; - segbuf -> recsink; - segbuf -> segsink [style=dashed]; - - // ════════ ④ QueueingRecordSink (worker task) — POST13zerocopy ════════ - subgraph cluster_queue { - label="④ QueueingRecordSink — pump ↔ decoder decoupling"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - qbuf [label="pump-side batch\nVec>\nbatch_size = 64", fillcolor="#4D3A28", shape=parallelogram]; - qchan [label="unbounded mpsc\nVec batches\nsoft cap → yield_now", fillcolor="#4D3A28", shape=parallelogram]; - qwrk [label="worker task\ndrains batches,\nfires on_idle ticks", fillcolor="#4D3A28"]; - qerr [label="shared err slot\nsurfaces back to pump", fillcolor="#4D3A28", shape=note]; - qbuf -> qchan -> qwrk; - qwrk -> qerr [style=dashed, color="#B58B86"]; - } - recsink -> qbuf [lhead=cluster_queue]; - - // ════════ ⑤ decode + xact buffer (gated on shadow apply) ════════ - subgraph cluster_decode { - label="⑤ decode + xact buffer (worker task — gated on shadow apply)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - catgate [label="BufferingDecoderSink\nrelation_at(rfn, lsn)\nwait_for_replay(at_lsn)", fillcolor="#4D4128"]; - decoder [label="heap_decoder\nINSERT / DELETE /\nUPDATE / HOT / TRUNCATE\nTier-1 fixed-width\nTier-2 length-prefixed\nReplica-identity matrix", fillcolor="#4D4128"]; - xactbuf [label="XactBuffer + spill v6\nHeap / Chunk / ToastDelete / Raw / dict\ndirty tree + SMGR markers + commit stash\nSubxactTracker\nbudget = work_mem", fillcolor="#4D4128"]; - drain [label="transaction reorder\nplan (side-effect-free: detoast + route +\nraw decode) then execute sealed plan\nwait for earlier data before table changes", fillcolor="#4D4128"]; - catgate -> decoder -> xactbuf -> drain; - } - qwrk -> catgate [lhead=cluster_decode]; - - // ════════ ⑥ emitter pipeline ════════ - subgraph cluster_ch { - label="⑥ emitter pipeline — shared path for ClickHouse and metrics-only runs"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - cdecode [label="decode pool ×M\nplanned envelopes (descriptor + route ride)\noracle columns stay raw\nchunk 1024 rows / 4 MiB", fillcolor="#5D4628"]; - tableenc [label="InsertBatcher\nTableEncoder per table\nNative columns +\n_lsn / _xid / _commit_ts / _is_deleted\nrow/byte budget + deadline seal", fillcolor="#5D4628"]; - chwire [label="inserter pool ×N\nBlockBuilder over sealed batch\noracle Native block spliced in\none complete INSERT each\nsend_with_retry + reconnect", fillcolor="#5D4628"]; - nulltail [label="metrics-only tail\nno ClickHouse connection\nacknowledge immediately", fillcolor="#4D3A28", shape=note]; - ackcol [label="ack collector\npublish latest position after\nevery earlier commit completes", fillcolor="#4D3A28", shape=note]; - toastio [label="ToastResolver / CH mirrors\nput births+tombstones before publish\nfetch pre-window values\ntruncate / retire / rewrite O−B", fillcolor="#5D4628"]; - cdecode -> tableenc -> chwire; - chwire -> ackcol [style=dotted, color="#b380b0", label="Acked after\nEndOfStream"]; - cdecode -> nulltail [style=dashed, color="#BF8C5F", constraint=false, label="metrics only"]; - nulltail -> ackcol [style=dotted, color="#b380b0", constraint=false, label="done"]; - } - drain -> cdecode [color="#BF8C5F", penwidth=2, label="DecodeJob"]; - drain -> toastio [color="#BF8C5F", style=dashed, label="new_rows + barriers"]; - toastio -> cdecode [color="#CBA85E", style=dashed, label="as-of fetch"]; - - // ════════ ⑦ ShadowCatalog cache + schema-event channel ════════ - subgraph cluster_catalog { - label="⑦ ShadowCatalog cache + SchemaEvent channel"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - lru [label="LRU 4096\nby_filenode / by_oid\n+ generation counter", fillcolor="#4D4D28"]; - libpq [label="tokio_postgres::Client\nSELECT pg_class /\npg_attribute / pg_type", fillcolor="#4D4D28"]; - inval [label="invalidation_drain\n(tokio task)\nbump generation\non catalog write", fillcolor="#4D4D28"]; - evtx [label="schema_event_tx\nAdded / Changed / Dropped\n(unbounded mpsc)", fillcolor="#4D4D28", shape=parallelogram]; - lru -> libpq [style=dashed, label="miss"]; - inval -> lru [style=dashed, color="#CBA85E"]; - lru -> evtx [style=dashed, color="#CBA85E", label="diff vs prior"]; - } - - // ════════ ⑦b DDL applicator + type bridge ════════ - subgraph cluster_ddl { - label="⑦b DDL applicator (separate CH TCP)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - bridge [label="type_bridge\nRelAttr → CH type\n(reject widen for now)", fillcolor="#5D4628"]; - ddlapp [label="DdlApplicator\n(ch_ddl.rs)\nCREATE / ADD COL /\nRENAME / DROP TABLE", fillcolor="#5D4628"]; - bridge -> ddlapp [style=dashed]; - } - evtx -> ddlapp [style=dashed, color="#CBA85E", label="drain at commit"]; - ddlapp -> ch [style=dashed, color="#BF8C5F", constraint=false, label="DDL SQL\n(inside reorder barrier,\n own CH connection)"]; - - catgate -> lru [style=dashed, color="#CBA85E", constraint=false, label="lookup"]; - track -> inval [style=dashed, color="#CBA85E"]; - evtx -> xactbuf [style=dashed, color="#CBA85E", constraint=false, label="stamp on xid"]; - - // ════════ ⑧ walsender server (tokio task) ════════ - subgraph cluster_walsender { - label="⑧ walsender server (tokio task)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - listener [label="accept(unix / TCP)\nIDENTIFY_SYSTEM +\nSTART_REPLICATION", fillcolor="#5D3F40"]; - sendq [label="per-conn send queue\n'w' XLogData @ record\n'k' keepalive on idle", fillcolor="#5D3F40"]; - stat [label="rx 'r' standby status\nflush_lsn / apply_lsn\n(min across conns,\n apply-lag metric)", fillcolor="#5D3F40"]; - listener -> sendq [style=dashed]; - } - bytesink -> sendq [color="#BD8183", penwidth=2]; - stat -> catgate [style=dashed, color="#BD8183", constraint=false, label="apply_lsn → unblock gate"]; - - // ════════ ⑨ on-disk caches ════════ - subgraph cluster_disk { - label="⑨ on-disk caches"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - outdir [label="out/\n16 MiB filtered\n+ manifest.json", fillcolor="#4D3850", shape=note]; - spill [label="{spill}/xid--.bin\nappend-only\n[tag u8 | len u32 | body]", fillcolor="#4D3850", shape=note]; - manifest [label="{spill}/manifest.toml\nsource identity + safe restart state\nheld behind open work", fillcolor="#4D3850", shape=note]; - retires [label="{spill}/toast_retires.toml\nsaved mirror retirements", fillcolor="#4D3850", shape=note]; - } - - // ════════ External edges ════════ - src -> feed [color="#A1A9CC", penwidth=2]; - sendq -> shd [color="#BD8183", penwidth=2, label="hot wire (ms)"]; - shd -> stat [style=dashed, color="#BD8183", constraint=false]; - libpq -> shd [color="#CBA85E", dir=both, arrowtail=open, label="libpq SELECT"]; - segsink -> outdir; - outdir -> shd [style=dashed, color="#6E6963", label="restore_command"]; - xactbuf -> spill [style=dashed, label="evict largest xact"]; - ackcol -> manifest [style=dotted, color="#b380b0", label="status loop saves\ndurable progress"]; - chwire -> ch [color="#BF8C5F", penwidth=2]; - toastio -> ch [color="#BF8C5F", style=dashed, label="pg_toast_"]; - drain -> retires [color="#6E6963", style=dashed, label="toast DROP"]; - manifest -> toastio [color="#6E6963", style=dashed, label="saved cleanup point"]; - retires -> toastio [color="#6E6963", style=dashed, label="pending retirements"]; - - // ════════ Anchor sinks at the bottom rank ════════ - { rank=sink; shd; ch; } - - // ════════ Legend ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - -
node fill — role
source PG / ingress
walshadow filter / queue (sync)
walshadow output sinks (walsender, segment)
walshadow decoder + xact buffer
walshadow ShadowCatalog + schema-event tx
walshadow walsender server
walshadow CH pipeline tail + DdlApplicator
on-disk artifact
shadow Postgres
edge colour
━━source replication frame
━━walsender wire (hot path)
━━libpq catalog query / schema event
━━CH Native (rows + DDL SQL)
···ack + manifest durability
┄┄filesystem (restore_command)
- >]; - ch -> legend [style=invis]; -} diff --git a/architecture/internals.svg b/architecture/internals.svg deleted file mode 100644 index f51258c1..00000000 --- a/architecture/internals.svg +++ /dev/null @@ -1,813 +0,0 @@ - - - - - - -internals - -walshadow internals — pipeline stages, taps & caches - -cluster_ingress - -① ingress (main loop, tokio) - - -cluster_filter - -② filter pipeline (sync, record cadence) - - -cluster_fanout - -③ CompositeRecordSink fan-out (pump task, sync — ordering matters) - - -cluster_queue - -④ QueueingRecordSink — pump ↔ decoder decoupling - - -cluster_decode - -⑤ decode + xact buffer (worker task — gated on shadow apply) - - -cluster_ch - -⑥ emitter pipeline — shared path for ClickHouse and metrics-only runs - - -cluster_catalog - -⑦ ShadowCatalog cache + SchemaEvent channel - - -cluster_ddl - -⑦b DDL applicator (separate CH TCP) - - -cluster_walsender - -⑧ walsender server  (tokio task) - - -cluster_disk - -⑨ on-disk caches - - - -src - - -source PG - - - -feed - -SourceFeed::pump -wal-rus ReplicationConn -IDENTIFY_SYSTEM + -START_REPLICATION PHYSICAL - - - -src->feed - - - - - -shd - - -shadow PG -walreceiver + catalog -(pg_last_wal_replay_lsn) - - - -stat - -rx 'r' standby status -flush_lsn / apply_lsn -(min across conns, - apply-lag metric) - - - -shd->stat - - - - - -ch - - -ClickHouse - - - -legend - - - -node fill — role - - - -source PG / ingress - - - -walshadow filter / queue (sync) - - - -walshadow output sinks (walsender, segment) - - - -walshadow decoder + xact buffer - - - -walshadow ShadowCatalog + schema-event tx - - - -walshadow walsender server - - - -walshadow CH pipeline tail + DdlApplicator - - - -on-disk artifact - - - -shadow Postgres - - -edge colour - -━━ - -source replication frame - -━━ - -walsender wire (hot path) - -━━ - -libpq catalog query / schema event - -━━ - -CH Native (rows + DDL SQL) - -··· - -ack + manifest durability - -┄┄ - -filesystem (restore_command) - - - - -chunks - -WalChunk stream -(start_lsn, server_wal_end, - bytes) - - - -feed->chunks - - - - - -walker - -StreamingWalker -page-state machine -record stitch across pages - - - -chunks->walker - - - - - -decide - -Filter::decide -Keep catalog -Drop user-heap -Empty → reclass - - - -walker->decide - - - - - -track - -CatalogTracker -relfilenode set -(relmap + pg_class writes) - - - -inval - -invalidation_drain -(tokio task) -bump generation -on catalog write - - - -track->inval - - - - - -decide->track - - - - - - -rewrite - -rewrite::noop_replace -in-place at byte_ranges -+ CRC32C recompute - - - -decide->rewrite - - - - - -segbuf - - -16 MiB segment buffer -(rewritten image) - - - -rewrite->segbuf - - - - - -bytesink - -❶ ShadowStreamSink -on_record_bytes -(stays on pump) - - - -segbuf->bytesink - - - - - -recsink - -❷ on_record → -QueueingRecordSink -(clones + enqueues, - returns immediately) - - - -segbuf->recsink - - - - - -segsink - -❸ DirSegmentSink -on_segment -(at 16 MiB boundary) - - - -segbuf->segsink - - - - - -sendq - -per-conn send queue -'w' XLogData @ record -'k' keepalive on idle - - - -bytesink->sendq - - - - - -qbuf - -pump-side batch -Vec<Record<'static>> -batch_size = 64 - - - -recsink->qbuf - - - - - -outdir - - - -out/<seg> -16 MiB filtered -+ manifest.json - - - -segsink->outdir - - - - - -qchan - -unbounded mpsc -Vec<Record> batches -soft cap → yield_now - - - -qbuf->qchan - - - - - -qwrk - -worker task -drains batches, -fires on_idle ticks - - - -qchan->qwrk - - - - - -qerr - - - -shared err slot -surfaces back to pump - - - -qwrk->qerr - - - - - -catgate - -BufferingDecoderSink -relation_at(rfn, lsn) -wait_for_replay(at_lsn) - - - -qwrk->catgate - - - - - -decoder - -heap_decoder -INSERT / DELETE / -UPDATE / HOT / TRUNCATE -Tier-1 fixed-width -Tier-2 length-prefixed -Replica-identity matrix - - - -catgate->decoder - - - - - -lru - -LRU 4096 -by_filenode / by_oid -+ generation counter - - - -catgate->lru - - -lookup - - - -xactbuf - -XactBuffer + spill v6 -Heap / Chunk / ToastDelete / Raw / dict -dirty tree + SMGR markers + commit stash -SubxactTracker -budget = work_mem - - - -decoder->xactbuf - - - - - -drain - -transaction reorder -plan (side-effect-free: detoast + route + -raw decode) then execute sealed plan -wait for earlier data before table changes - - - -xactbuf->drain - - - - - -spill - - - -{spill}/xid-<xid>-<lsn>.bin -append-only -[tag u8 | len u32 | body] - - - -xactbuf->spill - - -evict largest xact - - - -cdecode - -decode pool ×M -planned envelopes (descriptor + route ride) -oracle columns stay raw -chunk 1024 rows / 4 MiB - - - -drain->cdecode - - -DecodeJob - - - -toastio - -ToastResolver / CH mirrors -put births+tombstones before publish -fetch pre-window values -truncate / retire / rewrite O−B - - - -drain->toastio - - -new_rows + barriers - - - -retires - - - -{spill}/toast_retires.toml -saved mirror retirements - - - -drain->retires - - -toast DROP - - - -tableenc - -InsertBatcher -TableEncoder per table -Native columns + -_lsn / _xid / _commit_ts / _is_deleted -row/byte budget + deadline seal - - - -cdecode->tableenc - - - - - -nulltail - - - -metrics-only tail -no ClickHouse connection -acknowledge immediately - - - -cdecode->nulltail - - -metrics only - - - -chwire - -inserter pool ×N -BlockBuilder over sealed batch -oracle Native block spliced in -one complete INSERT each -send_with_retry + reconnect - - - -tableenc->chwire - - - - - -chwire->ch - - - - - -ackcol - - - -ack collector -publish latest position after -every earlier commit completes - - - -chwire->ackcol - - -Acked after -EndOfStream - - - -nulltail->ackcol - - -done - - - -manifest - - - -{spill}/manifest.toml -source identity + safe restart state -held behind open work - - - -ackcol->manifest - - -status loop saves -durable progress - - - -toastio->ch - - -pg_toast_<relid> - - - -toastio->cdecode - - -as-of fetch - - - -libpq - -tokio_postgres::Client -SELECT pg_class / -pg_attribute / pg_type - - - -lru->libpq - - -miss - - - -evtx - -schema_event_tx -Added / Changed / Dropped -(unbounded mpsc) - - - -lru->evtx - - -diff vs prior - - - -libpq->shd - - - -libpq SELECT - - - -inval->lru - - - - - -evtx->xactbuf - - -stamp on xid - - - -ddlapp - -DdlApplicator -(ch_ddl.rs) -CREATE / ADD COL / -RENAME / DROP TABLE - - - -evtx->ddlapp - - -drain at commit - - - -bridge - -type_bridge -RelAttr → CH type -(reject widen for now) - - - -bridge->ddlapp - - - - - -ddlapp->ch - - -DDL SQL -(inside reorder barrier, - own CH connection) - - - -listener - -accept(unix / TCP) -IDENTIFY_SYSTEM + -START_REPLICATION - - - -listener->sendq - - - - - -sendq->shd - - -hot wire (ms) - - - -stat->catgate - - -apply_lsn → unblock gate - - - -outdir->shd - - -restore_command - - - -manifest->toastio - - -saved cleanup point - - - -retires->toastio - - -pending retirements - - - diff --git a/architecture/ops.dot b/architecture/ops.dot deleted file mode 100644 index 909d8202..00000000 --- a/architecture/ops.dot +++ /dev/null @@ -1,124 +0,0 @@ -// walshadow — manifest durability + slot-advance feedback loop -// Status loop gathers progress, saves manifest, then publishes cleanup -// point and source feedback. -// -// regeneration spec: -// sources of truth: plans/ops.md · plans/TOAST.md · src/source/manifest.rs · src/toast/toast_retire.rs · src/ops/{preflight,retention,metrics}.rs · src/bin/stream.rs -// subsumes: plans/ops.md § "Manifest" + "Slot advance" + apply-lag metric -// quality bar: -// - persist → publish ordering reads as one durability path -// - slot-advance edge clearly back to source PG, not confused with metrics export -// - preflight noted as boot-only (not in tick loop) -// shared style: palette.md -digraph ops { - rankdir=TB; - compound=true; - newrank=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow — save restart state before cleanup or source feedback", fontsize=14, splines=spline, nodesep=0.4, ranksep=0.8, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ═════ External endpoints ═════ - shd [label="shadow PostgreSQL\nreceive + replay progress", fillcolor="#3D4128", shape=cylinder]; - src [label="source PostgreSQL\nreplication slot", fillcolor="#3D3D54", shape=cylinder]; - prom [label="Prometheus scrape\n/metrics", fillcolor="#3D3D54", shape=cylinder]; - toastmirror [label="ClickHouse TOAST mirrors", fillcolor="#4D4128", shape=cylinder]; - - // ═════ A. LSN inputs (left column, fed by xact/segment/wire) ═════ - subgraph cluster_inputs { - label="progress sources"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - feed [label="source connection\nidentity + received WAL position", fillcolor="#3D3D54"]; - segsink [label="filtered WAL archive\nsave segment before reporting progress", fillcolor="#4D3340"]; - xact [label="transaction pipeline\nopen transactions + durable progress", fillcolor="#4D4128"]; - sweeper [label="shadow replay monitor\ntrack recovery progress", fillcolor="#4D3A28"]; - walsrv [label="shadow receiver feedback\ntrack safe handoff point", fillcolor="#5D3F40"]; - } - - // ═════ B. status_loop orchestrator (centre column) ═════ - subgraph cluster_status { - label="periodic status loop"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - snap [label="collect source, archive, shadow,\ntransaction, and ClickHouse progress", fillcolor="#4D3A28"]; - floor [label="choose safe restart point\nnever pass saved WAL or\nan unfinished transaction", fillcolor="#4D3A28"]; - apply [label="cap source feedback at\nshadow + ClickHouse progress", fillcolor="#4D3A28"]; - mwrite [label="save manifest.toml\nusing atomic replacement", fillcolor="#4D3A28"]; - publish [label="share saved restart point\nwith cleanup tasks", fillcolor="#4D3A28"]; - triple [label="build source feedback\nwrite / flush / apply positions", fillcolor="#4D3A28"]; - sendst [label="send feedback\nto source replication slot", fillcolor="#3D3D54"]; - snap -> floor; - floor -> mwrite [color="#b380b0", style=dotted, penwidth=2]; - mwrite -> publish [color="#b380b0", style=dotted, penwidth=2]; - snap -> apply; - apply -> triple; - triple -> sendst; - } - - // ═════ C. Sinks (right column) ═════ - manifest_toml [label="manifest.toml\nsource identity + restart state", fillcolor="#4D3850", shape=note]; - retire_toml [label="toast_retires.toml\nsaved mirror retirements", fillcolor="#4D3850", shape=note]; - retire_flush [label="apply mirror retirements\nonce restart can no longer replay drop", fillcolor="#4D3A28"]; - - outdir [label="out/\nfiltered 16 MiB WAL\n+ manifest.json", fillcolor="#4D3850", shape=note]; - - trim [label="trim old filtered WAL\nkeep shadow recovery window", fillcolor="#4D3A28"]; - - metrics [label="metrics snapshot\nexport progress + apply lag", fillcolor="#4D4128"]; - - // ═════ Wiring ═════ - // shadow PG sits left of the inputs cluster, feeds sweeper + walsrv - shd -> sweeper [style=dashed, color="#CBA85E", label="replay query"]; - shd -> walsrv [style=dashed, color="#BD8183", label="receiver feedback"]; - - // Inputs → status_loop snapshot (all converge on `snap`) - feed -> snap [label="source"]; - segsink -> snap [label="saved WAL"]; - xact -> snap [label="transactions"]; - sweeper -> snap [label="shadow replay"]; - walsrv -> snap [label="shadow receive", color="#BD8183"]; - - // Manifest durability path - mwrite -> manifest_toml [color="#b380b0", style=dotted, penwidth=2, label="save"]; - xact -> retire_toml [color="#6E6963", style=dashed, label="record drop"]; - publish -> retire_flush [color="#b380b0", style=dotted, label="safe point"]; - retire_toml -> retire_flush [color="#6E6963", style=dashed, label="pending"]; - retire_flush -> toastmirror [color="#BF8C5F", style=dashed, label="wipe mirror"]; - - // Slot advance path - sendst -> src [color="#A1A9CC", penwidth=2, label="replication feedback"]; - - // Retention sweeper uses independent shadow-recovery cut - sweeper -> trim [color="#6E6963", style=dashed]; - trim -> outdir [color="#6E6963", style=dashed, label="remove old files"]; - - // Metrics tap (off main flow but ranks below snap so it points down with gravity) - snap -> metrics [style=dashed, color="#CBA85E", label="progress + apply lag"]; - metrics -> prom [color="#A1A9CC", label="HTTP /metrics"]; - - // Rank pinning so external endpoints anchor each side - { rank=same; shd; feed; } - { rank=same; src; manifest_toml; retire_toml; } - - // Legend - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - -
node fill — role
source PG / ingress
walshadow status / retention
walshadow segment sink
walshadow xact buffer / metrics
walshadow walsender server
on-disk artifact
shadow Postgres
edge colour
━━source replication ('r' standby) / Prom scrape
━━walsender wire ('r' from shadow)
━━libpq catalog query / metrics tap
···manifest persist → floor publish
┄┄filesystem (segment trim)
- >]; - manifest_toml -> legend [style=invis]; -} diff --git a/architecture/ops.svg b/architecture/ops.svg deleted file mode 100644 index 9097f539..00000000 --- a/architecture/ops.svg +++ /dev/null @@ -1,403 +0,0 @@ - - - - - - -ops - -walshadow — save restart state before cleanup or source feedback - -cluster_inputs - -progress sources - - -cluster_status - -periodic status loop - - - -shd - - -shadow PostgreSQL -receive + replay progress - - - -sweeper - -shadow replay monitor -track recovery progress - - - -shd->sweeper - - -replay query - - - -walsrv - -shadow receiver feedback -track safe handoff point - - - -shd->walsrv - - -receiver feedback - - - -src - - -source PostgreSQL -replication slot - - - -prom - - -Prometheus scrape -/metrics - - - -toastmirror - - -ClickHouse TOAST mirrors - - - -feed - -source connection -identity + received WAL position - - - -snap - -collect source, archive, shadow, -transaction, and ClickHouse progress - - - -feed->snap - - -source - - - -segsink - -filtered WAL archive -save segment before reporting progress - - - -segsink->snap - - -saved WAL - - - -xact - -transaction pipeline -open transactions + durable progress - - - -xact->snap - - -transactions - - - -retire_toml - - - -toast_retires.toml -saved mirror retirements - - - -xact->retire_toml - - -record drop - - - -sweeper->snap - - -shadow replay - - - -trim - -trim old filtered WAL -keep shadow recovery window - - - -sweeper->trim - - - - - -walsrv->snap - - -shadow receive - - - -floor - -choose safe restart point -never pass saved WAL or -an unfinished transaction - - - -snap->floor - - - - - -apply - -cap source feedback at -shadow + ClickHouse progress - - - -snap->apply - - - - - -metrics - -metrics snapshot -export progress + apply lag - - - -snap->metrics - - -progress + apply lag - - - -mwrite - -save manifest.toml -using atomic replacement - - - -floor->mwrite - - - - - -triple - -build source feedback -write / flush / apply positions - - - -apply->triple - - - - - -publish - -share saved restart point -with cleanup tasks - - - -mwrite->publish - - - - - -manifest_toml - - - -manifest.toml -source identity + restart state - - - -mwrite->manifest_toml - - -save - - - -retire_flush - -apply mirror retirements -once restart can no longer replay drop - - - -publish->retire_flush - - -safe point - - - -sendst - -send feedback -to source replication slot - - - -triple->sendst - - - - - -sendst->src - - -replication feedback - - - -legend - - - -node fill — role - - - -source PG / ingress - - - -walshadow status / retention - - - -walshadow segment sink - - - -walshadow xact buffer / metrics - - - -walshadow walsender server - - - -on-disk artifact - - - -shadow Postgres - - -edge colour - -━━ - -source replication ('r' standby) / Prom scrape - -━━ - -walsender wire ('r' from shadow) - -━━ - -libpq catalog query / metrics tap - -··· - -manifest persist → floor publish - -┄┄ - -filesystem (segment trim) - - - - -retire_toml->retire_flush - - -pending - - - -retire_flush->toastmirror - - -wipe mirror - - - -outdir - - - -out/<seg> -filtered 16 MiB WAL -+ manifest.json - - - -trim->outdir - - -remove old files - - - -metrics->prom - - -HTTP /metrics - - - diff --git a/architecture/oracle.dot b/architecture/oracle.dot deleted file mode 100644 index fbcd5875..00000000 --- a/architecture/oracle.dot +++ /dev/null @@ -1,126 +0,0 @@ -// walshadow, sealed-batch Native oracle -// Columns walshadow does not decode cross a unix socket to shadow PG's -// preloaded worker, which returns one partial ClickHouse Native block per -// sealed batch. Request atomicity and the splice stay visible. -// -// regeneration spec: -// sources of truth: plans/oracle.md · pgext/ · src/ops/oracle.rs · src/emit/pipeline/inserter.rs -// subsumes: plans/oracle.md § routing + protocol v2 + splicing + failure semantics -// quality bar: -// - one request per sealed batch reads as one canonical orange edge -// - the splice (local slabs + decoded columns into one block) stays legible -// - failure paths never reach ClickHouse -// shared style: palette.md -digraph oracle { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow oracle, sealed-batch Native columns", fontsize=14, splines=spline, nodesep=0.35, ranksep=0.9, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ ① decoder hand-off ════════ - subgraph cluster_decoder { - label="① decoder — ColumnValue (heap_decoder.rs)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - local [label="local matrix\nfixed-width / bytea / text /\nnumeric / inet / interval / json\nlocal_matrix_covers() == true", fillcolor="#4D4128"]; - pend [label="ColumnValue::PgPending\n{ type_oid, raw }\njsonb / array / hstore / range /\ndomain / enum / vendor", fillcolor="#5D3F40"]; - pendtx [label="ColumnValue::PgPendingText\n{ type_oid, text }\nattmissingval, canonical PG text\nnot a physical body", fillcolor="#5D3F40"]; - wkt [label="render_ext_columns\nPostGIS 2-D point -> WKT\nin-tree, matched on type_name\n(typoutput would give HEXEWKB)", fillcolor="#5D3F40"]; - } - - // ════════ ② plan + accumulate ════════ - subgraph cluster_plan { - label="② emit plan — decided once per column, never per row"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - choose [label="ColumnEncoding::choose\nOracle when the local matrix\nmisses the source attribute,\nor the target is Array / Map /\nJSON / Object", fillcolor="#4D4128"]; - slab [label="local ColumnBuf\nFixed / String /\nNullableFixed / NullableString", fillcolor="#4D4128"]; - ocol [label="ColumnBuf::Oracle\none cell per row:\nDefault | DiskRaw |\nTextInput | Literal\nsize-capped below the frame", fillcolor="#5D3F40"]; - } - - // ════════ ③ inserter, one request per sealed batch ════════ - subgraph cluster_inserter { - label="③ inserter (pipeline/inserter.rs) — before send_query"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - encode [label="Oracle::encode_batch\nrow-major request:\nall metadata, then all cells\none per InsertBatch", fillcolor="#5D3F40"]; - verify [label="validate response\none block, clean EOF,\nBlock::validate,\nrows / cols / names /\ncanonical type strings", fillcolor="#5D3F40"]; - splice [label="BlockBuilder\nappend local roots +\nappend_column decoded roots +\nsynthetic _lsn/_xid/_commit_ts\nblock outlives every retry", fillcolor="#5D4628"]; - stats [label="OracleStats\nblocks / rows / cells /\nconversion_errors / errors\n\nBridgeStats\nup / requests / latency /\nreconnects / native_bytes", fillcolor="#5D3F40", shape=note]; - } - - client [label="Bridge\ntokio::net::UnixStream\nHELLO version gate (proto 2)\nENCODE_NATIVE frame\nredial + one replay on\ntransport failure", fillcolor="#4D4D28"]; - - // ════════ ④ shadow PG + preloaded worker ════════ - subgraph cluster_shadow { - label="④ shadow PG"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - worker [label="walshadow background worker\n(pgext/worker.c)\nunix socket, mode 0600\none transaction per request\nno per-cell subtransaction:\nthe request is atomic", fillcolor="#3D4128"]; - native [label="pgext/native.c\nws_reconstruct_datum ×4 branches:\n typlen=-1 varlena\n typlen=-2 cstring\n typbyval fixed (memcpy)\n fixed by-ref (palloc)\nsource adapters: hstore matrix,\nmulti-arg casts\npgch_append_datum per cell", fillcolor="#3D4128"]; - writer [label="pgch_writer -> chc_block_write\nnull maps, array offsets,\nmap tuples, JSON markers\nstraight into the response\nStringInfo, local framing", fillcolor="#3D4128"]; - env [label="pinned output environment\nTimeZone = UTC\nDateStyle = ISO, MDY\nIntervalStyle = postgres\nextra_float_digits = 1\nbytea_output = hex", fillcolor="#3D4128", shape=note]; - } - - // ════════ ⑤ pgext build artifact ════════ - subgraph cluster_pgext { - label="⑤ pgext/ (PGXS build)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - artifact [label="walshadow.so\nnative.o + overlay.o + worker.o\nvendor/pg-clickhouse-c (pinned,\nunmodified, with clickhouse-c)\nno control file, no SQL script,\nno pg_proc row", fillcolor="#4D3850", shape=note]; - } - - // ════════ ⑥ terminal states ════════ - subgraph cluster_out { - label="⑥ outcome"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - ch [label="ClickHouse INSERT\nsend_query + send_data +\nsend_data_end, then ack", fillcolor="#5D4628"]; - fail [label="batch fails before any query\nconversion / semantic / transport\nseq stays unacknowledged,\nreplayed on restart", fillcolor="#5D4628", shape=note]; - } - - // ─── main spine ─── - local -> choose [color="#A1A9CC", weight=10]; - pend -> choose [color="#A1A9CC", penwidth=2, weight=10]; - pendtx -> choose [color="#A1A9CC", weight=5]; - wkt -> choose [color="#A1A9CC", style=dashed, label="literal cell", weight=5]; - choose -> slab [color="#A1A9CC", label="Local"]; - choose -> ocol [color="#A1A9CC", penwidth=2, label="Oracle"]; - ocol -> encode [color="#CBA85E", penwidth=2, label="sealed batch", weight=10]; - encode -> client [color="#CBA85E", penwidth=2, label="one request\nper batch", weight=10]; - client -> worker [color="#CBA85E", penwidth=2, dir=both, arrowtail=open, lhead=cluster_shadow, label="framed unix\nsocket request", weight=10]; - encode -> verify [color="#CBA85E", penwidth=2, label="Native bytes"]; - verify -> splice [color="#BF8C5F", penwidth=2, label="decoded column tree,\nborrowed not copied"]; - slab -> splice [color="#BF8C5F", label="owned slabs"]; - splice -> ch [color="#BF8C5F", penwidth=2]; - - // ─── shadow internals ─── - worker -> native [color="#6E6963", label="cell loop, error\ncontext = column + row"]; - native -> writer [color="#6E6963"]; - native -> env [style=dashed, color="#6E6963", constraint=false]; - - // ─── pgext preload edge ─── - artifact -> worker [style=dashed, color="#6E6963", lhead=cluster_shadow, ltail=cluster_pgext, label="$libdir or build tree\nshared_preload_libraries\n+ walshadow.* GUCs"]; - - // ─── failure fan-out ─── - worker -> fail [color="#8F5D5D", style=dashed, constraint=false, label="one bad Datum\naborts the request"]; - verify -> fail [color="#8F5D5D", style=dashed, constraint=false, label="wrong schema or\nmalformed block"]; - client -> fail [color="#8F5D5D", style=dotted, constraint=false, label="transport, past\nretry budget"]; - - // ─── stats fan-in ─── - encode -> stats [style=dashed, color="#CBA85E", constraint=false]; - client -> stats [style=dashed, color="#CBA85E", constraint=false]; - - // ─── Legend ─── - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - -
node fill
local decode + plan
oracle (src/ops/oracle.rs)
bridge client
shadow PG worker
pgext/ build artifact
final block / outcome
edge colour
━━value → encoding choice
━━unix socket (MAIN wire)
━━block assembly
┄┄failure, before any CH query
┄┄pgext preload (filesystem)
ownership
PostgreSQL owns Datum interpretation,
pg-clickhouse-c owns PG→CH conversion,
clickhouse-c owns Native format.
walshadow never reads an array element,
hstore pair, null map or nested offset.
atomicity
The writer has request lifetime and no
per-cell rollback, so one unconvertible
Datum fails the whole batch. No partial
block reaches the daemon, no ClickHouse
query starts, no row is acknowledged.
Nothing falls back to a substituted value.
- >]; - worker -> legend [style=invis]; -} diff --git a/architecture/oracle.svg b/architecture/oracle.svg deleted file mode 100644 index 2437034a..00000000 --- a/architecture/oracle.svg +++ /dev/null @@ -1,472 +0,0 @@ - - - - - - -oracle - -walshadow oracle, sealed-batch Native columns - -cluster_decoder - -① decoder — ColumnValue (heap_decoder.rs) - - -cluster_plan - -② emit plan — decided once per column, never per row - - -cluster_inserter - -③ inserter (pipeline/inserter.rs) — before send_query - - -cluster_shadow - -④ shadow PG - - -cluster_pgext - -⑤ pgext/ (PGXS build) - - -cluster_out - -⑥ outcome - - - -local - -local matrix -fixed-width / bytea / text / -numeric / inet / interval / json -local_matrix_covers() == true - - - -choose - -ColumnEncoding::choose -Oracle when the local matrix -misses the source attribute, -or the target is Array / Map / -JSON / Object - - - -local->choose - - - - - -pend - -ColumnValue::PgPending -{ type_oid, raw } -jsonb / array / hstore / range / -domain / enum / vendor - - - -pend->choose - - - - - -pendtx - -ColumnValue::PgPendingText -{ type_oid, text } -attmissingval, canonical PG text -not a physical body - - - -pendtx->choose - - - - - -wkt - -render_ext_columns -PostGIS 2-D point -> WKT -in-tree, matched on type_name -(typoutput would give HEXEWKB) - - - -wkt->choose - - -literal cell - - - -slab - -local ColumnBuf -Fixed / String / -NullableFixed / NullableString - - - -choose->slab - - -Local - - - -ocol - -ColumnBuf::Oracle -one cell per row: -Default | DiskRaw | -TextInput | Literal -size-capped below the frame - - - -choose->ocol - - -Oracle - - - -splice - -BlockBuilder -append local roots + -append_column decoded roots + -synthetic _lsn/_xid/_commit_ts -block outlives every retry - - - -slab->splice - - -owned slabs - - - -encode - -Oracle::encode_batch -row-major request: -all metadata, then all cells -one per InsertBatch - - - -ocol->encode - - -sealed batch - - - -verify - -validate response -one block, clean EOF, -Block::validate, -rows / cols / names / -canonical type strings - - - -encode->verify - - -Native bytes - - - -stats - - - -OracleStats -blocks / rows / cells / -conversion_errors / errors -BridgeStats -up / requests / latency / -reconnects / native_bytes - - - -encode->stats - - - - - -client - -Bridge -tokio::net::UnixStream -HELLO version gate (proto 2) -ENCODE_NATIVE frame -redial + one replay on -transport failure - - - -encode->client - - -one request -per batch - - - -verify->splice - - -decoded column tree, -borrowed not copied - - - -fail - - - -batch fails before any query -conversion / semantic / transport -seq stays unacknowledged, -replayed on restart - - - -verify->fail - - -wrong schema or -malformed block - - - -ch - -ClickHouse INSERT -send_query + send_data + -send_data_end, then ack - - - -splice->ch - - - - - -client->stats - - - - - -worker - -walshadow background worker -(pgext/worker.c) -unix socket, mode 0600 -one transaction per request -no per-cell subtransaction: -the request is atomic - - - -client->worker - - - -framed unix -socket request - - - -client->fail - - -transport, past -retry budget - - - -native - -pgext/native.c -ws_reconstruct_datum ×4 branches: -  typlen=-1 varlena -  typlen=-2 cstring -  typbyval fixed (memcpy) -  fixed by-ref (palloc) -source adapters: hstore matrix, -multi-arg casts -pgch_append_datum per cell - - - -worker->native - - -cell loop, error -context = column + row - - - -worker->fail - - -one bad Datum -aborts the request - - - -legend - - - -node fill - - - -local decode + plan - - - -oracle (src/ops/oracle.rs) - - - -bridge client - - - -shadow PG worker - - - -pgext/ build artifact - - - -final block / outcome - - -edge colour - -━━ - -value → encoding choice - -━━ - -unix socket (MAIN wire) - -━━ - -block assembly - -┄┄ - -failure, before any CH query - -┄┄ - -pgext preload (filesystem) - - -ownership - -PostgreSQL owns Datum interpretation, -pg-clickhouse-c owns PG→CH conversion, -clickhouse-c owns Native format. -walshadow never reads an array element, -hstore pair, null map or nested offset. - - -atomicity - -The writer has request lifetime and no -per-cell rollback, so one unconvertible -Datum fails the whole batch. No partial -block reaches the daemon, no ClickHouse -query starts, no row is acknowledged. -Nothing falls back to a substituted value. - - - - -writer - -pgch_writer -> chc_block_write -null maps, array offsets, -map tuples, JSON markers -straight into the response -StringInfo, local framing - - - -native->writer - - - - - -env - - - -pinned output environment -TimeZone = UTC -DateStyle = ISO, MDY -IntervalStyle = postgres -extra_float_digits = 1 -bytea_output = hex - - - -native->env - - - - - -artifact - - - -walshadow.so -native.o + overlay.o + worker.o -vendor/pg-clickhouse-c (pinned, -unmodified, with clickhouse-c) -no control file, no SQL script, -no pg_proc row - - - -artifact->worker - - -$libdir or build tree -shared_preload_libraries -+ walshadow.* GUCs - - - diff --git a/architecture/overview.dot b/architecture/overview.dot deleted file mode 100644 index 8403bf9e..00000000 --- a/architecture/overview.dot +++ /dev/null @@ -1,44 +0,0 @@ -// walshadow — 30-second system view -// 5 boxes, 6 wires. For deeper pipeline detail, see internals.dot. -digraph overview { - rankdir=TB; - graph [fontname="Helvetica", labelloc="t", label="walshadow — source Postgres → ClickHouse, with shadow Postgres as catalog-replay sidecar", fontsize=14, splines=spline, nodesep=0.6, ranksep=1.1, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=12, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=10, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ─── 5 actors ─── - src [label="source\nPostgres\n(primary,\nwal_level=replica)", fillcolor="#3D3D54", shape=cylinder]; - wal [label="walshadow-stream\n\nfilters WAL (drop user-heap,\nkeep catalog) → rewrites CRC,\ndecodes heap → emits to CH,\nDDL events → CH ALTER/CREATE", - fillcolor="#4D3A28", width=2.6]; - arc [label="out/\nfiltered 16 MiB\nWAL + manifest", fillcolor="#4D3850", shape=note]; - shd [label="shadow\nPostgres\n(continuous recovery,\nschema/catalog mirror)", fillcolor="#3D4128", shape=cylinder]; - ch [label="ClickHouse\nReplacingMergeTree(_lsn)", fillcolor="#4D4128", shape=cylinder]; - - // ─── 6 wires ─── - src -> wal [label="① physical replication\nCopyData 'w'", color="#A1A9CC", penwidth=2]; - wal -> shd [label="② walsender wire\n'w' XLogData (ms cadence)", color="#BD8183", penwidth=2]; - wal -> shd [label="③ libpq SELECT\ncatalog cache fill", color="#CBA85E", dir=both, arrowtail=open, constraint=false]; - wal -> arc [label="④ fsync segment\n+ manifest", color="#6E6963"]; - arc -> shd [label="⑤ restore_command\n(archive fallback)", style=dashed, color="#6E6963", constraint=false]; - wal -> ch [label="⑥ CH Native blocks\natomic-seal INSERT\n(buffered multi-xact,\n budget / deadline)", color="#BF8C5F", penwidth=2]; - - // ─── Legend ─── - legend [shape=plaintext, label=< - - - - - - - - - - - - - - -
wire
①source → walshadow: physical replication (START_REPLICATION PHYSICAL)
②walshadow → shadow: walsender wire, record cadence (hot path)
③walshadow ↔ shadow: libpq SELECT on cache miss (pg_class / pg_attribute)
④walshadow → on-disk: fsynced filtered WAL segments + manifests
⑤on-disk → shadow: restore_command archive fallback when walsender stalls
⑥walshadow → ClickHouse: Native protocol, rows buffered across xacts, sealed as complete INSERTs (budget/deadline)
where to go next
internals.svg — pipeline stages, taps, caches
shadow_communication.svg — channels ②③⑤ in detail
timeline_bootstrap.svg — greenfield bring-up
timeline_streaming.svg — steady-state record path
timeline_restart.svg — restart scenarios
- >]; - ch -> legend [style=invis]; -} diff --git a/architecture/overview.svg b/architecture/overview.svg index 260abc94..e3497d06 100644 --- a/architecture/overview.svg +++ b/architecture/overview.svg @@ -1,164 +1,84 @@ - - - - - - -overview - -walshadow — source Postgres → ClickHouse, with shadow Postgres as catalog-replay sidecar - - -src - - -source -Postgres -(primary, -wal_level=replica) - - - -wal - -walshadow-stream -filters WAL (drop user-heap, -keep catalog) → rewrites CRC, -decodes heap → emits to CH, -DDL events → CH ALTER/CREATE - - - -src->wal - - -① physical replication -CopyData 'w' - - - -arc - - - -out/<seg> -filtered 16 MiB -WAL + manifest - - - -wal->arc - - -④ fsync segment -+ manifest - - - -shd - - -shadow -Postgres -(continuous recovery, -schema/catalog mirror) - - - -wal->shd - - -② walsender wire -'w' XLogData (ms cadence) - - - -wal->shd - - - -③ libpq SELECT -catalog cache fill - - - -ch - - -ClickHouse -ReplacingMergeTree(_lsn) - - - -wal->ch - - -⑥ CH Native blocks -atomic-seal INSERT -(buffered multi-xact, - budget / deadline) - - - -arc->shd - - -⑤ restore_command -(archive fallback) - - - -legend - - - -wire - -① - -source → walshadow: physical replication (START_REPLICATION PHYSICAL) - -② - -walshadow → shadow: walsender wire, record cadence (hot path) - -③ - -walshadow ↔ shadow: libpq SELECT on cache miss (pg_class / pg_attribute) - -④ - -walshadow → on-disk: fsynced filtered WAL segments + manifests - -⑤ - -on-disk → shadow: restore_command archive fallback when walsender stalls - -⑥ - -walshadow → ClickHouse: Native protocol, rows buffered across xacts, sealed as complete INSERTs (budget/deadline) - - -where to go next - -internals.svg - — pipeline stages, taps, caches - -shadow_communication.svg - — channels ②③⑤ in detail - -timeline_bootstrap.svg - — greenfield bring-up - -timeline_streaming.svg - — steady-state record path - -timeline_restart.svg - — restart scenarios - - - + + Streaming topology + Physical WAL splits into original records for row processing and filtered bytes for shadow replay. CatalogCapture reads shadow at held replay boundaries, persists DescriptorLog history and attaches schema events to transactions. Bounded record queue feeds transaction buffering and the parallel commit pipeline, which writes ClickHouse. + + + + + + + + + + walshadow-stream process + + Source PostgreSQL + Physical replication slot + + physical WAL + + SourceFeed → WalStream + StreamingWalker + Filter + Keep catalogs; no-op user heap + + Filtered WAL sinks + ShadowStreamSink → sender + DirSegmentSink → out/<seg> + + WAL + + Shadow PostgreSQL + + walreceiver + startup replay + + Catalog interface + worker SCAN + libpq + + WAL + + QueueingRecordSink + + + + + original records + + BufferingDecoderSink + XactBuffer: heaps + raw WAL + Commit / abort, spill to disk + + bounded mpsc + + CatalogCapture + Hold WAL at schema boundary + Read exact replay position + + SCAN + + DescriptorLog + Schema history by WAL position + + + lookup + + ReorderSink → parallel commit pipeline + DecodeJob → decode[M] → InsertBatcher → inserter[N] + SchemaEvent / TRUNCATE use an ordered DDL barrier + + + descriptors + + events + + ClickHouse + Native INSERT connections + Separate DDL connection + + CatalogCapture runs on pump; queued row processing owns BufferingDecoderSink and ReorderSink diff --git a/architecture/palette.md b/architecture/palette.md deleted file mode 100644 index 8eb2c268..00000000 --- a/architecture/palette.md +++ /dev/null @@ -1,77 +0,0 @@ -Shared style for all architecture/*.dot. Not for humans - -## graph defaults - -``` -bgcolor #272623 -fontcolor #ECE1D7 -fontname Helvetica -fontsize 14 (graph), 10 (node), 9 (edge) -splines spline -``` - -## cluster defaults - -``` -style rounded,filled -color #4c4641 -fillcolor #34302c -fontcolor #ECE1D7 -``` - -## node fills (actor) - -| actor | fill | -|---|---| -| source PG / ingress | #3D3D54 | -| walshadow filter, queue, ingress (sync) | #4D3A28 | -| walshadow output sinks (walsender, segment) | #4D3340 | -| walshadow decoder + xact buffer | #4D4128 | -| walshadow ShadowCatalog + schema event | #4D4D28 | -| walshadow walsender server | #5D3F40 | -| walshadow CH emitter + DdlApplicator | #5D4628 | -| on-disk artifact | #4D3850 (shape=note) | -| shadow Postgres | #3D4128 | -| ClickHouse | #4D4128 | - -## edge colors (channel) - -| channel | color | style | -|---|---|---| -| source replication frame | #A1A9CC | solid | -| walsender wire (hot path) | #BD8183 | solid; thick for primary | -| libpq catalog query | #CBA85E | solid (bidir on cache fill) | -| CH Native rows | #BF8C5F | solid | -| CH DDL SQL (separate TCP) | #BF8C5F | dashed | -| manifest durability / ack events | #b380b0 | dotted | -| filesystem / restore_command | #6E6963 | dashed | -| error / sidecar feedback | #B58B86 | dashed, constraint=false | - -## sidecar edges - -cross-cutting cache feedback, error feedback, gate dependencies must use `constraint=false` + `style=dashed` so main column stays straight - -## legend - -every diagram ends in `legend [shape=plaintext, label=]`. Required rows: -- node-fill key (only fills present in diagram) -- edge-color key (only channels present) -- optional: domain-specific subtable (e.g. rmgr keep policy for filter) - -last edge in graph: ` -> legend [style=invis];` to anchor - -## render - -``` -cd architecture && dot -Tsvg .dot -o .svg && dot -Tpng .dot -o .png -``` - -## quality bar - -human review pass for these properties: -- nodes don't overlap -- edges don't form spaghetti (curves cross at most 1-2 times in well-trafficked area) -- legend readable at 100% zoom -- clusters don't span >2x sibling widths -- no orphan nodes outside clusters -- thick edges (`penwidth=2,3`) reserved for primary channels diff --git a/architecture/recovery.svg b/architecture/recovery.svg new file mode 100644 index 00000000..4b3c5ba5 --- /dev/null +++ b/architecture/recovery.svg @@ -0,0 +1,71 @@ + + Restart floor, persistence and cleanup + Status loop combines unfinished transaction positions, completed ClickHouse work and fsynced filtered WAL to compute a conservative aligned restart floor. It persists manifest.toml before publishing that floor to descriptor garbage collection, TOAST retirements and source feedback. Restart validates source identity and obtains WAL from source or configured archive. + + + + + + + + + XactBuffer + Oldest unfinished transaction + Pending commit / spill state + + AckCollector + Contiguous durable work + emitter_ack_lsn + + Segment fsync task + Durable filtered WAL + durable_lsn + + + + + + + Status loop / resolved_floor + Protect unfinished transactions + Align down, clamp to durable filtered WAL + + + manifest.toml + Source identity, timeline, saved restart floor + + Restart + Validate source identity + Read saved floor + Source → archive + → source fallback + Replay unfinished work + + + persist before publish + + Publish saved floor + resume_floor / GC watch / source feedback + + + + + + + DescriptorLog GC + Discard obsolete history + below saved replay floor + + TOAST retirements + toast_retires.toml ledger + Retire after saved floor + + Source slot feedback + write / flush / apply + Also cap by shadow progress + diff --git a/architecture/shadow.dot b/architecture/shadow.dot deleted file mode 100644 index aa35d0a0..00000000 --- a/architecture/shadow.dot +++ /dev/null @@ -1,100 +0,0 @@ -// walshadow, ShadowCatalog descriptor assembly from bridge projections -// -// regeneration spec: -// sources of truth: plans/shadow.md · src/catalog/shadow_catalog.rs · src/ops/bridge.rs · pgext/overlay.c -// subsumes: plans/shadow.md § ShadowCatalog + One descriptor definition + Uncommitted DDL + Reconnect resilience -// differentiates: shadow_communication.dot draws transport channels; this draws descriptor read and assembly paths -// quality bar: -// - committed and overlay bridge paths stay visible -// - replay movement fails closed -// - replay pin spans every worker scan and oid chunk -// shared style: palette.md -digraph shadow_catalog { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="ShadowCatalog, pinned catalog projections → one descriptor definition", fontsize=14, splines=spline, nodesep=0.45, ranksep=0.65, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - subgraph cluster_catalog { - label="ShadowCatalog (src/catalog/shadow_catalog.rs)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - { rank=same; - committed [label="committed APIs\nfetch_descriptors_batch(oids)\nfetch_all_descriptors()\ndescriptor_by_name / toast\nscope = oids | eligible", fillcolor="#4D4D28"]; - overlay [label="overlay API\nfetch_overlay_descriptors\noids + top_xid + boundary\nbridge required", fillcolor="#4D4D28"]; - replay [label="replay gate API\nwait_for_replay(target)\nmonotone last observed LSN\npoll until target or timeout", fillcolor="#4D4D28"]; - } - - scan [label="scan_rows, worker source\ntop_xid = 0: committed view\ntop_xid ≠ 0: own-xid overlay\npg_class → pg_attribute → pg_index\nnamespace + type name maps", fillcolor="#4D4D28"]; - - pin [label="scan_pinned invariant\nfirst SCAN fixes replay LSN\nor caller supplies boundary\nevery catalog + oid chunk\nmust start and end at same LSN", fillcolor="#4D4D28", shape=note]; - - libpq [label="tokio-postgres client\nunix socket, long-lived\nensure_open + one reconnect retry\nreset last_replay_lsn on reconnect", fillcolor="#4D4D28"]; - - rows [label="DescriptorRows\nclass / attrs / indexes\nnamespace + type maps\nreplay_lsn", fillcolor="#4D4D28", shape=folder]; - - assemble [label="DescriptorRows::assemble\nreject duplicate oid or attnum\npreserve dropped attribute slots\nchoose PK + replica identity\nresolve reltablespace 0\nuse raw relfilenode", fillcolor="#4D4D28"]; - - result [label="RelDescriptor\nrfn + oid + toast oid\nname + kind + persistence\nreplident + attributes\n\ncommitted: replay_lsn + Vec\noverlay: Vec", fillcolor="#4D4D28", shape=parallelogram]; - - committed -> scan [label="top_xid = 0", color="#CBA85E", penwidth=2]; - overlay -> scan [label="top_xid + boundary", color="#CBA85E", penwidth=2]; - replay -> libpq [label="pg_last_wal_replay_lsn()", color="#CBA85E"]; - scan -> pin [style=dashed, color="#B58B86", constraint=false]; - scan -> libpq [label="committed namespace/type names\nDB oid + default tablespace", color="#CBA85E", dir=both, arrowtail=open]; - scan -> rows [label="parsed SCAN rows", color="#CBA85E"]; - rows -> assemble [color="#CBA85E", penwidth=2]; - assemble -> result [color="#CBA85E", penwidth=2]; - } - - bridge [label="Bridge\nframed unix socket\nSCAN request\nchunk at MAX_SCAN_OIDS", fillcolor="#4D4D28"]; - - subgraph cluster_shadow { - label="shadow PG"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - worker [label="walshadow worker\nSCAN under SnapshotAny\nown-xid visibility\nreplay LSN before + after", fillcolor="#3D4128"]; - - catalogs [label="pg_class / pg_attribute /\npg_index / pg_namespace /\npg_type / pg_database\n+ replay position", fillcolor="#3D4128", shape=cylinder]; - - worker -> catalogs [label="direct catalog rows", color="#6E6963", dir=both, arrowtail=open]; - } - - scan -> bridge [label="one SCAN per catalog / chunk", color="#CBA85E", penwidth=2, dir=both, arrowtail=open]; - bridge -> worker [label="worker protocol", color="#CBA85E", penwidth=2, dir=both, arrowtail=open, lhead=cluster_shadow]; - libpq -> catalogs [label="SQL + pg_last_wal_replay_lsn()", color="#CBA85E", penwidth=2, dir=both, arrowtail=open, lhead=cluster_shadow]; - - capture [label="CatalogCapture\nboundary hold requires\nreturned LSN == next_lsn\ndiff historical predecessor\nderive SchemaEvent", fillcolor="#4D4128"]; - - direct [label="direct consumers\nboot seed / opt-in dispatch /\nbackup TOAST lookup", fillcolor="#4D4128"]; - - log [label="DescriptorLog\nappend durable batch\nown descriptor history", fillcolor="#4D3850", shape=note]; - - xbuf [label="XactBuffer\nstamp SchemaEvent on xid\ncommit-order drain", fillcolor="#4D4128"]; - - result -> capture [label="boundary fetch", color="#CBA85E", penwidth=2]; - result -> direct [label="non-boundary fetch", color="#CBA85E"]; - capture -> log [label="batch", color="#b380b0", style=dotted, penwidth=2]; - capture -> xbuf [label="Added | Changed | Dropped", color="#b380b0", style=dotted]; - - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - -
node fill, role
ShadowCatalog + bridge client
shadow PG worker + catalogs
capture and descriptor consumers
durable descriptor log
edge color
━━descriptor query + assembly path
┄┄replay-pin invariant
┈┈descriptor durability + schema-event handoff
source policy
committedworker only; ReplayMismatch fails read
overlayworker only; ReplayMismatch fails read
ownership
ShadowCatalog has no cache, invalidation state, or event channel. DescriptorLog owns history; CatalogCapture derives events.
- >]; - log -> legend [style=invis]; -} diff --git a/architecture/shadow.svg b/architecture/shadow.svg deleted file mode 100644 index d86abc9c..00000000 --- a/architecture/shadow.svg +++ /dev/null @@ -1,349 +0,0 @@ - - - - - - -shadow_catalog - -ShadowCatalog, pinned catalog projections → one descriptor definition - -cluster_catalog - -ShadowCatalog (src/catalog/shadow_catalog.rs) - - -cluster_shadow - -shadow PG - - - -committed - -committed APIs -fetch_descriptors_batch(oids) -fetch_all_descriptors() -descriptor_by_name / toast -scope = oids | eligible - - - -scan - -scan_rows, worker source -top_xid = 0: committed view -top_xid ≠ 0: own-xid overlay -pg_class → pg_attribute → pg_index -namespace + type name maps - - - -committed->scan - - -top_xid = 0 - - - -overlay - -overlay API -fetch_overlay_descriptors -oids + top_xid + boundary -bridge required - - - -overlay->scan - - -top_xid + boundary - - - -replay - -replay gate API -wait_for_replay(target) -monotone last observed LSN -poll until target or timeout - - - -libpq - -tokio-postgres client -unix socket, long-lived -ensure_open + one reconnect retry -reset last_replay_lsn on reconnect - - - -replay->libpq - - -pg_last_wal_replay_lsn() - - - -pin - - - -scan_pinned invariant -first SCAN fixes replay LSN -or caller supplies boundary -every catalog + oid chunk -must start and end at same LSN - - - -scan->pin - - - - - -scan->libpq - - - -committed namespace/type names -DB oid + default tablespace - - - -rows - -DescriptorRows -class / attrs / indexes -namespace + type maps -replay_lsn - - - -scan->rows - - -parsed SCAN rows - - - -bridge - -Bridge -framed unix socket -SCAN request -chunk at MAX_SCAN_OIDS - - - -scan->bridge - - - -one SCAN per catalog / chunk - - - -catalogs - - -pg_class / pg_attribute / -pg_index / pg_namespace / -pg_type / pg_database -+ replay position - - - -libpq->catalogs - - - -SQL + pg_last_wal_replay_lsn() - - - -assemble - -DescriptorRows::assemble -reject duplicate oid or attnum -preserve dropped attribute slots -choose PK + replica identity -resolve reltablespace 0 -use raw relfilenode - - - -rows->assemble - - - - - -result - -RelDescriptor -rfn + oid + toast oid -name + kind + persistence -replident + attributes -committed: replay_lsn + Vec -overlay: Vec - - - -assemble->result - - - - - -capture - -CatalogCapture -boundary hold requires -returned LSN == next_lsn -diff historical predecessor -derive SchemaEvent - - - -result->capture - - -boundary fetch - - - -direct - -direct consumers -boot seed / opt-in dispatch / -backup TOAST lookup - - - -result->direct - - -non-boundary fetch - - - -worker - -walshadow worker -SCAN under SnapshotAny -own-xid visibility -replay LSN before + after - - - -bridge->worker - - - -worker protocol - - - -worker->catalogs - - - -direct catalog rows - - - -log - - - -DescriptorLog -append durable batch -own descriptor history - - - -capture->log - - -batch - - - -xbuf - -XactBuffer -stamp SchemaEvent on xid -commit-order drain - - - -capture->xbuf - - -Added | Changed | Dropped - - - -legend - - - -node fill, role - - - -ShadowCatalog + bridge client - - - -shadow PG worker + catalogs - - - -capture and descriptor consumers - - - -durable descriptor log - - -edge color - -━━ - -descriptor query + assembly path - -┄┄ - -replay-pin invariant - -┈┈ - -descriptor durability + schema-event handoff - - -source policy - -committed - -worker only; ReplayMismatch fails read - -overlay - -worker only; ReplayMismatch fails read - - -ownership - -ShadowCatalog has no cache, invalidation state, or event channel. DescriptorLog owns history; CatalogCapture derives events. - - - - diff --git a/architecture/shadow_communication.dot b/architecture/shadow_communication.dot deleted file mode 100644 index 60b77f6e..00000000 --- a/architecture/shadow_communication.dot +++ /dev/null @@ -1,84 +0,0 @@ -// walshadow ↔ shadow PG — three communication channels -// 1. libpq queries (catalog cache fill) -// 2. walsender wire (record-cadence WAL push) -// 3. restore_command (archive fallback, segment cadence) -digraph shadow_comm { - rankdir=TB; - graph [fontname="Helvetica", labelloc="t", label="walshadow ↔ shadow PG — three communication channels", fontsize=14, splines=spline, nodesep=0.6, ranksep=1.0, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=10, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ──────── walshadow side ──────── - subgraph cluster_w { - label="walshadow-stream daemon"; style="solid"; color="#A1A9CC"; - bgcolor="#34302c"; - - catalog [label="ShadowCatalog\nRelDescriptor LRU\n+ tokio_postgres client\n+ schema_event_tx", fillcolor="#3D3D54"]; - inval [label="invalidation_drain task\ngeneration counter", fillcolor="#3D3D54"]; - - sender [label="ShadowStreamSink + walsender\nlistener (unix / TCP)\nframe encoder", fillcolor="#4D3340"]; - segwrt [label="DirSegmentSink\nfiltered segments + manifest", fillcolor="#4D3850"]; - - bootstrap [label="Bootstrap orchestrator\nspawn_greenfield_bootstrap\n(catalog seed + BASE_BACKUP)", fillcolor="#4D4128"]; - } - - // ──────── on-disk archive ──────── - outdir [label="out/\n16 MiB filtered\nsegments + manifests", fillcolor="#5A3D5A", shape=note]; - dataok [label="shadow data dir\npopulated by bootstrap\n(catalog files landed)", fillcolor="#5A3D5A", shape=note]; - - // ──────── shadow side ──────── - subgraph cluster_s { - label="shadow PG (continuous recovery)"; style="solid"; color="#869461"; - bgcolor="#34302c"; - - walrecv [label="walreceiver\nprimary_conninfo=walshadow\nflush/apply LSN", fillcolor="#3D4128"]; - restorer[label="restore_command\ncp out/%f %p", fillcolor="#3D4128"]; - shadcat [label="pg_class /\npg_attribute /\npg_type / pg_namespace", fillcolor="#3D4128"]; - pglog [label="redo apply\npg_last_wal_replay_lsn()", fillcolor="#3D4128"]; - } - - // ──────── Channel 1: libpq (catalog reads) ──────── - catalog -> shadcat [label="① libpq SELECT\n(cache fill on miss)\nport=55434", color="#CBA85E", dir=both, arrowtail=open, penwidth=2]; - - // ──────── Channel 2: walsender wire (hot path) ──────── - sender -> walrecv [label="② 'w' XLogData frames\nrecord cadence (ms)\nstreaming-replication protocol", color="#BD8183", penwidth=3]; - walrecv -> sender [label="② 'r' standby status\nflush_lsn / apply_lsn", color="#BD8183", style=dashed]; - - // ──────── Channel 3: restore_command (archive fallback) ──────── - segwrt -> outdir [label="fsync segment + manifest", color="#6E6963"]; - outdir -> restorer[label="③ filesystem read\non walreceiver disconnect\nor end-of-WAL", color="#6E6963", penwidth=2]; - restorer-> pglog [label="apply segment", color="#6E6963"]; - walrecv -> pglog [label="apply 'w' frames", color="#BD8183"]; - - // ──────── Bootstrap channel (one-shot) ──────── - bootstrap -> dataok [label="④ catalog files\n(BASE_BACKUP landed)", color="#BF8C5F", style=dashed, penwidth=2]; - dataok -> pglog [label="initial state", color="#BF8C5F", style=dashed]; - - // ──────── Catalog invalidation feedback ──────── - pglog -> walrecv [label="advance\npg_last_wal_replay_lsn", style=dashed]; - inval -> catalog [label="bump generation\n(force cache miss)", style=dashed, color="#CBA85E"]; - - // ──────── Catalog gate dependency ──────── - walrecv -> catalog [label="apply_lsn drives\nrelation_at gate\n(unblocks decoder)", style=dotted, color="#A1A9CC", constraint=false]; - - // ──────── Legend ──────── - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - -
channels (edge color)
① ━━libpq queries (catalog cache fill, on miss)
② ━━walsender wire — 'w' XLogData @ record cadence (ms)
③ ━━restore_command — archive fallback (segment cadence)
④ ┄┄BASE_BACKUP land (one-shot, greenfield)
···catalog gate dependency (apply_lsn → relation_at)
derived flow (off ①)
Schema event flow: cache miss → schema diff → SchemaEvent → DdlApplicator → CH ALTER/CREATE/DROP. Stays inside walshadow; CH wire not drawn.
node fill
walshadow — ShadowCatalog (libpq client)
walshadow — walsender / sender side
walshadow — on-disk segments / shadow data dir
walshadow — bootstrap orchestrator
shadow Postgres receivers / catalog
- >]; - pglog -> legend [style=invis]; -} diff --git a/architecture/shadow_communication.svg b/architecture/shadow_communication.svg deleted file mode 100644 index 48aaecf3..00000000 --- a/architecture/shadow_communication.svg +++ /dev/null @@ -1,267 +0,0 @@ - - - - - - -shadow_comm - -walshadow ↔ shadow PG — three communication channels - -cluster_w - -walshadow-stream daemon - - -cluster_s - -shadow PG (continuous recovery) - - - -catalog - -ShadowCatalog -RelDescriptor LRU -+ tokio_postgres client -+ schema_event_tx - - - -shadcat - -pg_class / -pg_attribute / -pg_type / pg_namespace - - - -catalog->shadcat - - - -① libpq SELECT -(cache fill on miss) -port=55434 - - - -inval - -invalidation_drain task -generation counter - - - -inval->catalog - - -bump generation -(force cache miss) - - - -sender - -ShadowStreamSink + walsender -listener (unix / TCP) -frame encoder - - - -walrecv - -walreceiver -primary_conninfo=walshadow -flush/apply LSN - - - -sender->walrecv - - -② 'w' XLogData frames -record cadence (ms) -streaming-replication protocol - - - -segwrt - -DirSegmentSink -filtered segments + manifest - - - -outdir - - - -out/<seg> -16 MiB filtered -segments + manifests - - - -segwrt->outdir - - -fsync segment + manifest - - - -bootstrap - -Bootstrap orchestrator -spawn_greenfield_bootstrap -(catalog seed + BASE_BACKUP) - - - -dataok - - - -shadow data dir -populated by bootstrap -(catalog files landed) - - - -bootstrap->dataok - - -④ catalog files -(BASE_BACKUP landed) - - - -restorer - -restore_command -cp out/%f %p - - - -outdir->restorer - - -③ filesystem read -on walreceiver disconnect -or end-of-WAL - - - -pglog - -redo apply -pg_last_wal_replay_lsn() - - - -dataok->pglog - - -initial state - - - -walrecv->catalog - - -apply_lsn drives -relation_at gate -(unblocks decoder) - - - -walrecv->sender - - -② 'r' standby status -flush_lsn / apply_lsn - - - -walrecv->pglog - - -apply 'w' frames - - - -restorer->pglog - - -apply segment - - - -pglog->walrecv - - -advance -pg_last_wal_replay_lsn - - - -legend - - - -channels (edge color) - -① ━━ - -libpq queries (catalog cache fill, on miss) - -② ━━ - -walsender wire — 'w' XLogData @ record cadence (ms) - -③ ━━ - -restore_command — archive fallback (segment cadence) - -④ ┄┄ - -BASE_BACKUP land (one-shot, greenfield) - -··· - -catalog gate dependency (apply_lsn → relation_at) - - -derived flow (off ①) - -Schema event flow: cache miss → schema diff → SchemaEvent → DdlApplicator → CH ALTER/CREATE/DROP. Stays inside walshadow; CH wire not drawn. - - -node fill - - - -walshadow — ShadowCatalog (libpq client) - - - -walshadow — walsender / sender side - - - -walshadow — on-disk segments / shadow data dir - - - -walshadow — bootstrap orchestrator - - - -shadow Postgres receivers / catalog - - - - diff --git a/architecture/source.dot b/architecture/source.dot deleted file mode 100644 index ccccc890..00000000 --- a/architecture/source.dot +++ /dev/null @@ -1,119 +0,0 @@ -// walshadow — source pipeline detail (per plans/source.md) -// SourceFeed → StreamingWalker → CompositeRecordSink fan-out: -// ❶ ShadowStreamSink (bytes, walsender wire) -// ❷ QueueingRecordSink (worker, decoder) -// ❸ DirSegmentSink (16 MiB segment artifact) -// Emphasizes the pump-task / worker-task boundary at QueueingRecordSink. -// -// regeneration spec: -// sources of truth: plans/source.md · src/source_feed.rs · src/wal_stream.rs · src/queueing_record_sink.rs · src/walsender_server*.rs -// subsumes: plans/source.md § sink composition / fan-out / QueueingRecordSink -// quality bar: -// - "stays on pump" vs "worker task" labels visually distinct -// - ❶❷❸ ordering glyphs visible in fan-out nodes -// - listener / sendq / statrx triangle reads as one walsender entity -// shared style: palette.md -digraph source { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="source pipeline — SourceFeed → StreamingWalker → CompositeRecordSink fan-out", fontsize=14, splines=spline, nodesep=0.5, ranksep=0.7, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ External actors ════════ - src [label="source PG\nwal_level=logical", fillcolor="#3D3D54", shape=cylinder]; - shd [label="shadow PG\nwalreceiver\npg_last_wal_replay_lsn", fillcolor="#3D4128", shape=cylinder]; - dec [label="downstream\nBufferingDecoderSink\n+ ReorderSink\n→ CH pipeline", fillcolor="#4D4128", shape=cylinder]; - - // ════════ ① ingress (pump task) ════════ - subgraph cluster_ingress { - label="① ingress (pump task, tokio)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - repconn [label="ReplicationConn (wal-rus)\nIDENTIFY_SYSTEM +\nSTART_REPLICATION PHYSICAL\n+ TLS / SCRAM-SHA-256", fillcolor="#3D3D54"]; - feed [label="SourceFeed::pump\nnext_chunk frame loop,\n'k' keepalive absorb,\n'r' standby status @ 10s", fillcolor="#3D3D54"]; - chunks [label="WalChunk\n{ start_lsn, server_wal_end,\n data: &[u8] }", fillcolor="#3D3D54", shape=parallelogram]; - repconn -> feed -> chunks; - } - - // ════════ ② walker (pump task, sync, record cadence) ════════ - subgraph cluster_walker { - label="② WalStream::push (pump task, sync — record cadence)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - walker [label="StreamingWalker\npage-state machine,\nrecord stitch across pages,\n16 MiB buffer (one alloc)", fillcolor="#4D3A28"]; - drain [label="WalStream::drain_records\nFilter::decide (keep / drop) +\nnoop_replace + CRC32C +\nparsed.into_owned()", fillcolor="#4D3A28"]; - walker -> drain; - } - chunks -> walker [color="#A1A9CC", penwidth=2]; - - // ════════ ③ CompositeRecordSink fan-out (pump task, order matters) ════════ - subgraph cluster_fanout { - label="③ CompositeRecordSink fan-out (pump task — bytes before record before segment)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - bytesink [label="❶ ShadowStreamSink\non_wire_chunk\n(stays on pump task)", fillcolor="#4D3340"]; - metric [label="MetricsRecordSink\nsync counters", fillcolor="#4D3340"]; - qsink [label="❷ QueueingRecordSink\non_record\n(clones to 'static,\n enqueues, returns)", fillcolor="#4D3340"]; - segsink [label="❸ DirSegmentSink\non_segment\n@ 16 MiB boundary", fillcolor="#4D3340"]; - } - drain -> bytesink [color="#BD8183", penwidth=2, label="❶"]; - drain -> metric [style=dashed]; - drain -> qsink [label="❷"]; - drain -> segsink [style=dashed, label="❸"]; - - // ════════ ④ Queue boundary (pump → worker) ════════ - subgraph cluster_queue { - label="④ QueueingRecordSink — pump-task ↔ worker-task boundary"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - qbuf [label="pump-side batch\nVec>\nbatch_size = 64", fillcolor="#4D3A28", shape=parallelogram]; - qchan [label="unbounded mpsc\nin_flight AtomicU64,\nsoft_cap → yield_now", fillcolor="#4D3A28", shape=parallelogram]; - qwrk [label="worker task\ndrain, on_idle ticks,\non_idle_advance(lsn)", fillcolor="#4D3A28"]; - qerr [label="shared err slot\n(wait_for_replay timeout\n surfaces back to pump)", fillcolor="#4D3A28", shape=note]; - qbuf -> qchan -> qwrk; - qwrk -> qerr [style=dashed, color="#B58B86"]; - } - qsink -> qbuf [lhead=cluster_queue]; - qerr -> qsink [style=dashed, color="#B58B86", constraint=false, label="next on_record →\nErr (real root cause,\n daemon exits clean)"]; - - // ════════ ⑤ walsender server (own tokio task) ════════ - subgraph cluster_walsender { - label="⑤ walsender server (wal-rus server.rs — own tokio task)"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - listener [label="accept loop\n127.0.0.1 TCP +\nSO_REUSEADDR / unix\nStartupMessage replication=true\nIDENTIFY_SYSTEM cached", fillcolor="#5D3F40"]; - sendq [label="WalSenderConn send queue\nper-conn Vec\n'w' XLogData + 'k' keepalive\nslow-client cutoff", fillcolor="#5D3F40"]; - stat [label="decode_standby_status\nrx 'r' write/flush/apply\nShadowStreamState aggregate", fillcolor="#5D3F40"]; - listener -> sendq [style=dashed]; - } - bytesink -> sendq [color="#BD8183", penwidth=2, label="enqueue_framed\n(record cadence, ms)"]; - - // ════════ ⑥ on-disk artifact ════════ - outdir [label="out/\nfsynced 16 MiB +\nmanifest.json", fillcolor="#4D3850", shape=note]; - segsink -> outdir; - - // ════════ External edges ════════ - src -> repconn [color="#A1A9CC", penwidth=2, label="CopyData 'w'\nstreaming replication"]; - sendq -> shd [color="#BD8183", penwidth=2, label="'w' XLogData\nrecord cadence"]; - shd -> stat [style=dashed, color="#BD8183", constraint=false, label="'r' standby status"]; - outdir -> shd [style=dashed, color="#6E6963", constraint=false, label="restore_command\n(archive fallback)"]; - qwrk -> dec [color="#BF8C5F", penwidth=2, label="DecoderXactPair\n(batched records)"]; - - // ════════ Cross-cutting secondary edges ════════ - stat -> qwrk [style=dotted, color="#A1A9CC", constraint=false, label="apply_lsn →\nunblock wait_for_replay\n(catalog gate)"]; - - // ════════ Legend ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - -
node fill — role
source PG / ingress (SourceFeed, ReplicationConn)
walshadow filter / walker / queue (pump-side sync)
CompositeRecordSink fan-out sinks
walsender server (accept, send queue, status rx)
on-disk artifact (filtered segment + manifest)
shadow Postgres (walreceiver)
downstream decoder + CH emitter
edge colour — channel
━━source replication frame (CopyData 'w')
━━walsender wire (hot path, record cadence)
━━batched records → decoder / CH emitter
┄┄filesystem (restore_command archive fallback)
···apply_lsn feedback (catalog gate unblock)
┄┄worker error surfaced back to pump task
fan-out order (③, mandatory)
❶ bytes → walsender (advances shadow apply_lsn)
❷ record → queue → decoder (waits on apply_lsn at catalog gate)
❸ segment fires only at 16 MiB boundary
- >]; - dec -> legend [style=invis]; -} diff --git a/architecture/source.svg b/architecture/source.svg deleted file mode 100644 index bd5a5a0d..00000000 --- a/architecture/source.svg +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - -source - -source pipeline — SourceFeed → StreamingWalker → CompositeRecordSink fan-out - -cluster_ingress - -① ingress (pump task, tokio) - - -cluster_walker - -② WalStream::push (pump task, sync — record cadence) - - -cluster_fanout - -③ CompositeRecordSink fan-out (pump task — bytes before record before segment) - - -cluster_queue - -④ QueueingRecordSink — pump-task ↔ worker-task boundary - - -cluster_walsender - -⑤ walsender server (wal-rus server.rs — own tokio task) - - - -src - - -source PG -wal_level=logical - - - -repconn - -ReplicationConn (wal-rus) -IDENTIFY_SYSTEM + -START_REPLICATION PHYSICAL -+ TLS / SCRAM-SHA-256 - - - -src->repconn - - -CopyData 'w' -streaming replication - - - -shd - - -shadow PG -walreceiver -pg_last_wal_replay_lsn - - - -stat - -decode_standby_status -rx 'r' write/flush/apply -ShadowStreamState aggregate - - - -shd->stat - - -'r' standby status - - - -dec - - -downstream -BufferingDecoderSink -+ ReorderSink -→ CH pipeline - - - -legend - - - -node fill — role - - - -source PG / ingress (SourceFeed, ReplicationConn) - - - -walshadow filter / walker / queue (pump-side sync) - - - -CompositeRecordSink fan-out sinks - - - -walsender server (accept, send queue, status rx) - - - -on-disk artifact (filtered segment + manifest) - - - -shadow Postgres (walreceiver) - - - -downstream decoder + CH emitter - - -edge colour — channel - -━━ - -source replication frame (CopyData 'w') - -━━ - -walsender wire (hot path, record cadence) - -━━ - -batched records → decoder / CH emitter - -┄┄ - -filesystem (restore_command archive fallback) - -··· - -apply_lsn feedback (catalog gate unblock) - -┄┄ - -worker error surfaced back to pump task - - -fan-out order (③, mandatory) - -❶ bytes → walsender (advances shadow apply_lsn) -❷ record → queue → decoder (waits on apply_lsn at catalog gate) -❸ segment fires only at 16 MiB boundary - - - - -feed - -SourceFeed::pump -next_chunk frame loop, -'k' keepalive absorb, -'r' standby status @ 10s - - - -repconn->feed - - - - - -chunks - -WalChunk -{ start_lsn, server_wal_end, -  data: &[u8] } - - - -feed->chunks - - - - - -walker - -StreamingWalker -page-state machine, -record stitch across pages, -16 MiB buffer (one alloc) - - - -chunks->walker - - - - - -drain - -WalStream::drain_records -Filter::decide (keep / drop) + -noop_replace + CRC32C + -parsed.into_owned() - - - -walker->drain - - - - - -bytesink - -❶ ShadowStreamSink -on_wire_chunk -(stays on pump task) - - - -drain->bytesink - - -❶ - - - -metric - -MetricsRecordSink -sync counters - - - -drain->metric - - - - - -qsink - -❷ QueueingRecordSink -on_record -(clones to 'static, - enqueues, returns) - - - -drain->qsink - - -❷ - - - -segsink - -❸ DirSegmentSink -on_segment -@ 16 MiB boundary - - - -drain->segsink - - -❸ - - - -sendq - -WalSenderConn send queue -per-conn Vec<u8> -'w' XLogData + 'k' keepalive -slow-client cutoff - - - -bytesink->sendq - - -enqueue_framed -(record cadence, ms) - - - -qbuf - -pump-side batch -Vec<Record<'static>> -batch_size = 64 - - - -qsink->qbuf - - - - - -outdir - - - -out/<seg> -fsynced 16 MiB + -manifest.json - - - -segsink->outdir - - - - - -qchan - -unbounded mpsc -in_flight AtomicU64, -soft_cap → yield_now - - - -qbuf->qchan - - - - - -qwrk - -worker task -drain, on_idle ticks, -on_idle_advance(lsn) - - - -qchan->qwrk - - - - - -qwrk->dec - - -DecoderXactPair -(batched records) - - - -qerr - - - -shared err slot -(wait_for_replay timeout - surfaces back to pump) - - - -qwrk->qerr - - - - - -qerr->qsink - - -next on_record → -Err (real root cause, - daemon exits clean) - - - -listener - -accept loop -127.0.0.1 TCP + -SO_REUSEADDR / unix -StartupMessage replication=true -IDENTIFY_SYSTEM cached - - - -listener->sendq - - - - - -sendq->shd - - -'w' XLogData -record cadence - - - -stat->qwrk - - -apply_lsn → -unblock wait_for_replay -(catalog gate) - - - -outdir->shd - - -restore_command -(archive fallback) - - - diff --git a/architecture/timeline_bootstrap.dot b/architecture/timeline_bootstrap.dot deleted file mode 100644 index 7e6a482a..00000000 --- a/architecture/timeline_bootstrap.dot +++ /dev/null @@ -1,120 +0,0 @@ -// walshadow — greenfield bootstrap timeline -// 5 phase clusters top→bottom; node fill colour-codes the actor -digraph timeline_bootstrap { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow greenfield bootstrap — 5 phases", fontsize=14, splines=spline, nodesep=0.35, ranksep=0.5, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ═════ ① catalog seed ═════ - subgraph cluster_p1 { - label="① catalog seed"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - op1 [label="walshadow-stream\n--bootstrap-mode=direct\n--ch-config=…", fillcolor="#403a36"]; - sql1 [label="sidecar SQL on source\nSELECT oid, relfilenode\nFROM pg_class WHERE oid≥16384", fillcolor="#3D3D54"]; - map1 [label="CatalogTracker::seed_from_source\nbuilds CatalogMap", fillcolor="#4D3A28"]; - - op1 -> map1 [label="spawn"]; - map1 -> sql1 [label="libpq", color="#CBA85E", dir=both, arrowtail=open]; - } - - // ═════ ② BASE_BACKUP pump ═════ - subgraph cluster_p2 { - label="② BASE_BACKUP pump — MultiplexSink fan-out"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - bs2 [label="BackupSource\n(Direct | ObjectStore)\nbegin pump", fillcolor="#4D3A28"]; - src2 [label="BASE_BACKUP open\npg_export_snapshot()\nstart_lsn = X", fillcolor="#3D3D54"]; - lan2 [label="DiskLanderSink\ncatalog → shadow data dir\nuser-heap filenode Skip", fillcolor="#4D3A28"]; - walk2 [label="PageWalkSink\ndecode 8 KiB heap pages\nmain + pg_toast tuples with TID\n→ BackfillTuple (mpsc)", fillcolor="#4D3A28"]; - dsk2 [label="shadow data dir\ncatalog files landed", fillcolor="#4D3850", shape=note]; - leg2 [label="window WAL leg\ndaemon feed, sampled pre-backup\ndecode → insert tail", fillcolor="#4D3A28"]; - - bs2 -> src2 [label="replication protocol", color="#A1A9CC", dir=both, arrowtail=open]; - bs2 -> lan2 [label="MultiplexSink"]; - bs2 -> walk2 [label="MultiplexSink"]; - lan2 -> dsk2 [label="land file"]; - src2 -> leg2 [label="window WAL, live", color="#A1A9CC", dir=both, arrowtail=open]; - } - map1 -> bs2 [label="open BASE_BACKUP", lhead=cluster_p2, color="#A1A9CC"]; - - // ═════ ③ drain → CH (concurrent with ②) ═════ - subgraph cluster_p3 { - label="③ drain → ClickHouse (concurrent with ②)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - gate3 [label="visibility gate\nhint bits → inline\nunknown → spool\nresolve from pg_xact + patch", fillcolor="#4D3A28"]; - drain3 [label="pipeline::bootstrap::drain\nToastRows → mirror batch\nExternalToast → deferred\nflush chunks, fetch + resolve\nmap main rows; one seq per rfn flip", fillcolor="#4D3A28"]; - ch3 [label="ClickHouse\nTOAST mirrors: put before deferred fetch\nmain tables: shared insert tail\nbatcher + inserter ×N + ack collector\ntail.finish → all seqs durable", fillcolor="#4D4128"]; - - gate3 -> drain3 [label="tuples that were live\nat backup time"]; - drain3 -> ch3 [label="BatcherMsg::Row", color="#BF8C5F"]; - } - walk2 -> gate3 [label="BackfillTuple\n(bounded mpsc 256,\nbackpressures pump)", lhead=cluster_p3]; - leg2 -> ch3 [label="window commits at real _lsn\noutrank walked rows", color="#BF8C5F", lhead=cluster_p3]; - - // ═════ ④ shadow handoff ═════ - subgraph cluster_p4 { - label="④ shadow handoff"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - end4 [label="BASE_BACKUP end\nend_lsn = Y", fillcolor="#3D3D54"]; - out4 [label="BootstrapOutcome\n{start_lsn, end_lsn}", fillcolor="#4D3A28"]; - ctrl4 [label="pg_control landed last\n(barrier)", fillcolor="#4D3850", shape=note]; - conf4 [label="Shadow::enable_standby_recovery\nappend standby.signal +\nrestore_command +\nprimary_conninfo", fillcolor="#4D3A28"]; - files4 [label="postgresql.conf +\nstandby.signal", fillcolor="#4D3850", shape=note]; - listen4 [label="walsender listener up\n(barrier before shadow start)", fillcolor="#4D3A28"]; - start4 [label="Shadow::start (recovery mode)\nblock_in_place pg_ctl", fillcolor="#4D3A28"]; - shd4 [label="postmaster + walreceiver\nbegin replay", fillcolor="#3D4128"]; - - end4 -> out4 [label="end_lsn", dir=back, color="#A1A9CC"]; - out4 -> ctrl4 [style=invis]; - out4 -> conf4; - conf4 -> files4 [label="write"]; - conf4 -> listen4; - listen4 -> start4 [label="barrier"]; - start4 -> shd4 [label="pg_ctl start"]; - } - bs2 -> end4 [label="finish", lhead=cluster_p4, color="#A1A9CC"]; - - // ═════ ⑤ WAL streaming start ═════ - subgraph cluster_p5 { - label="⑤ WAL streaming start (→ steady-state)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - seed5 [label="use backup end as\nnew restart point", fillcolor="#4D3A28"]; - manifest5 [label="manifest.toml\nsaved restart state", fillcolor="#4D3850", shape=note]; - feed5 [label="SourceFeed open\nSTART_REPLICATION\nPHYSICAL ", fillcolor="#4D3A28"]; - repl5 [label="streaming protocol\nopens at end_lsn", fillcolor="#3D3D54"]; - apply5 [label="pg_last_wal_replay_lsn ≥ end_lsn\nready for relation_at gate", fillcolor="#3D4128"]; - hot5 [label="hot streaming\n(pipeline: reorder → decode ×M\n→ same tail + DdlApplicator)\n→ timeline_streaming", fillcolor="#4D3A28", shape=cds]; - - seed5 -> feed5; - feed5 -> manifest5 [label="first status update", color="#b380b0", style=dotted]; - feed5 -> repl5 [label="START_REPLICATION", color="#A1A9CC", dir=both, arrowtail=open]; - feed5 -> hot5; - } - out4 -> seed5 [label="next phase", lhead=cluster_p5]; - shd4 -> apply5 [label="replay catches up", style=dashed, color="#869461"]; - - // ═════ Legend ═════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - -
node fill — actor
operator / CLI
source Postgres
walshadow-stream
on-disk artifact
shadow Postgres
ClickHouse
edge colour
━━physical replication protocol
━━libpq catalog query
━━ClickHouse Native blocks
┄┄shadow replay progress
- >]; - hot5 -> legend [style=invis]; -} diff --git a/architecture/timeline_bootstrap.svg b/architecture/timeline_bootstrap.svg deleted file mode 100644 index d7b30ca2..00000000 --- a/architecture/timeline_bootstrap.svg +++ /dev/null @@ -1,482 +0,0 @@ - - - - - - -timeline_bootstrap - -walshadow greenfield bootstrap — 5 phases - -cluster_p1 - -① catalog seed - - -cluster_p2 - -② BASE_BACKUP pump — MultiplexSink fan-out - - -cluster_p3 - -③ drain → ClickHouse  (concurrent with ②) - - -cluster_p4 - -④ shadow handoff - - -cluster_p5 - -⑤ WAL streaming start  (→ steady-state) - - - -op1 - -walshadow-stream ---bootstrap-mode=direct ---ch-config=… - - - -map1 - -CatalogTracker::seed_from_source -builds CatalogMap - - - -op1->map1 - - -spawn - - - -sql1 - -sidecar SQL on source -SELECT oid, relfilenode -FROM pg_class WHERE oid≥16384 - - - -map1->sql1 - - - -libpq - - - -bs2 - -BackupSource -(Direct | ObjectStore) -begin pump - - - -map1->bs2 - - -open BASE_BACKUP - - - -src2 - -BASE_BACKUP open -pg_export_snapshot() -start_lsn = X - - - -bs2->src2 - - - -replication protocol - - - -lan2 - -DiskLanderSink -catalog → shadow data dir -user-heap filenode Skip - - - -bs2->lan2 - - -MultiplexSink - - - -walk2 - -PageWalkSink -decode 8 KiB heap pages -main + pg_toast tuples with TID -→ BackfillTuple (mpsc) - - - -bs2->walk2 - - -MultiplexSink - - - -end4 - -BASE_BACKUP end -end_lsn = Y - - - -bs2->end4 - - -finish - - - -leg2 - -window WAL leg -daemon feed, sampled pre-backup -decode → insert tail - - - -src2->leg2 - - - -window WAL, live - - - -dsk2 - - - -shadow data dir -catalog files landed - - - -lan2->dsk2 - - -land file - - - -gate3 - -visibility gate -hint bits → inline -unknown → spool -resolve from pg_xact + patch - - - -walk2->gate3 - - -BackfillTuple -(bounded mpsc 256, -backpressures pump) - - - -ch3 - -ClickHouse -TOAST mirrors: put before deferred fetch -main tables: shared insert tail -batcher + inserter ×N + ack collector -tail.finish → all seqs durable - - - -leg2->ch3 - - -window commits at real _lsn -outrank walked rows - - - -drain3 - -pipeline::bootstrap::drain -ToastRows → mirror batch -ExternalToast → deferred -flush chunks, fetch + resolve -map main rows; one seq per rfn flip - - - -gate3->drain3 - - -tuples that were live -at backup time - - - -drain3->ch3 - - -BatcherMsg::Row - - - -out4 - -BootstrapOutcome -{start_lsn, end_lsn} - - - -end4->out4 - - -end_lsn - - - -ctrl4 - - - -pg_control landed last -(barrier) - - - - -conf4 - -Shadow::enable_standby_recovery -append standby.signal + -restore_command + -primary_conninfo - - - -out4->conf4 - - - - - -seed5 - -use backup end as -new restart point - - - -out4->seed5 - - -next phase - - - -files4 - - - -postgresql.conf + -standby.signal - - - -conf4->files4 - - -write - - - -listen4 - -walsender listener up -(barrier before shadow start) - - - -conf4->listen4 - - - - - -start4 - -Shadow::start (recovery mode) -block_in_place pg_ctl - - - -listen4->start4 - - -barrier - - - -shd4 - -postmaster + walreceiver -begin replay - - - -start4->shd4 - - -pg_ctl start - - - -apply5 - -pg_last_wal_replay_lsn ≥ end_lsn -ready for relation_at gate - - - -shd4->apply5 - - -replay catches up - - - -feed5 - -SourceFeed open -START_REPLICATION -PHYSICAL <end_lsn> - - - -seed5->feed5 - - - - - -manifest5 - - - -manifest.toml -saved restart state - - - -feed5->manifest5 - - -first status update - - - -repl5 - -streaming protocol -opens at end_lsn - - - -feed5->repl5 - - - -START_REPLICATION - - - -hot5 - -hot streaming -(pipeline: reorder → decode ×M -→ same tail + DdlApplicator) -→ timeline_streaming - - - -feed5->hot5 - - - - - -legend - - - -node fill — actor - - - -operator / CLI - - - -source Postgres - - - -walshadow-stream - - - -on-disk artifact - - - -shadow Postgres - - - -ClickHouse - - -edge colour - -━━ - -physical replication protocol - -━━ - -libpq catalog query - -━━ - -ClickHouse Native blocks - -┄┄ - -shadow replay progress - - - - diff --git a/architecture/timeline_restart.dot b/architecture/timeline_restart.dot deleted file mode 100644 index b8d8f10f..00000000 --- a/architecture/timeline_restart.dot +++ /dev/null @@ -1,98 +0,0 @@ -// walshadow — restart recovery: 3 scenarios as side-by-side phase columns -digraph timeline_restart { - rankdir=TB; - compound=true; - newrank=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow restart timelines — 3 scenarios, time flows top → bottom within each column", fontsize=14, splines=spline, nodesep=0.3, ranksep=0.45, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7", width=2.4]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ═════ A. clean restart (SIGTERM) ═════ - subgraph cluster_a { - label="A. clean restart (SIGTERM, saved restart point still available)"; - style="rounded,filled"; color="#A1A9CC"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - a0 [label="① SIGTERM", fillcolor="#403a36"]; - a1 [label="② save partial WAL segment", fillcolor="#4D3340"]; - a2 [label="③ drain queued work\nand finish ClickHouse writes", fillcolor="#4D3A28"]; - a3 [label="④ latest manifest remains\nconservative restart point", fillcolor="#4D3A28"]; - a4 [label="⑤ process exit", fillcolor="#403a36"]; - a5 [label="⑥ restart:\ncheck source identity\nload manifest.toml + toast_retires.toml\nresume from saved point", fillcolor="#4D3A28"]; - a6 [label="⑦ Shadow::start in recovery mode\n(if not already running)", fillcolor="#3D4128"]; - a7 [label="⑧ walsender listener up\nbefore walreceiver connects", fillcolor="#4D3340"]; - a8 [label="⑨ resume source WAL stream\nfrom saved point", fillcolor="#4D3A28"]; - a9 [label="⑩ records re-flow\nCH ReplacingMergeTree(_lsn) dedups", fillcolor="#4D4128"]; - - a0 -> a1 -> a2 -> a3 -> a4 -> a5 -> a6 -> a7 -> a8 -> a9; - } - - // ═════ B. kill -9 mid-stream ═════ - subgraph cluster_b { - label="B. kill -9 mid-stream (last saved manifest authoritative)"; - style="rounded,filled"; color="#CBA85E"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - b0 [label="① kill -9 (no drain)", fillcolor="#403a36"]; - b1 [label="② OS reaps process\nmemory lost; last complete manifest survives\npartial manifest write ignored", fillcolor="#4D3A28"]; - b2 [label="③ restart:\nclear transaction spill files\nkeep toast_retires.toml", fillcolor="#4D3A28"]; - b3 [label="④ check source identity\nresume from saved restart point\nflush safe mirror retirements", fillcolor="#4D3A28"]; - b4 [label="⑤ shadow PG recovers on its own\nwalreceiver retry timer", fillcolor="#3D4128"]; - b5 [label="⑥ shadow asks START_REPLICATION\nPHYSICAL ", fillcolor="#3D4128"]; - b6 [label="⑦ resume source from saved point\nresume shadow from its own progress", fillcolor="#4D3A28"]; - b7 [label="⑧ overlap records replayed\nCH dedups on _lsn\nshadow apply monotonic", fillcolor="#4D4128"]; - b8 [label="⑨ steady-state reached", fillcolor="#4D3A28"]; - - b0 -> b1 -> b2 -> b3 -> b4 -> b5 -> b6 -> b7 -> b8; - } - - // ═════ C. source recycled resume point → archive fallback ═════ - subgraph cluster_c { - label="C. restart recovery (source → archive → source)"; - style="rounded,filled"; color="#BD8183"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - c0 [label="① restart: initialized shadow (no incomplete marker)\n→ resume in place, never base backup\nresume_lsn = manifest end_lsn (after bootstrap)\nor --start-lsn", fillcolor="#403a36"]; - c1 [label="② START_REPLICATION at resume_lsn\n(stream.next_lsn)", fillcolor="#4D3A28"]; - c2 [label="③ transient drop → retry source once\n(primary_conninfo first)", fillcolor="#4D3A28"]; - c3 [label="④ fetch [backup] segment covering resume_lsn\n(full segment sliced from resume_lsn);\nreplay through live filter + decode sinks", fillcolor="#4D3340"]; - c4 [label="⑤ archive lacks next segment\nreconnect source at exact handoff (backoff)", fillcolor="#4D3A28"]; - c5 [label="⑥ source has handoff WAL\nresume live stream", fillcolor="#4D3A28"]; - c6 [label="⑥ 58P01 + archive can't cover\nexit; operator decides base refresh", fillcolor="#403a36"]; - c7 [label="⑦ steady-state reached\nCH ReplacingMergeTree(_lsn) dedups", fillcolor="#4D4128"]; - - c0 -> c1; - c1 -> c7 [label="available", color="#8FA875", fontcolor="#8FA875"]; - c1 -> c2 [label="drop", color="#CBA85E", fontcolor="#CBA85E"]; - c1 -> c3 [label="58P01", color="#BD8183", fontcolor="#BD8183"]; - c2 -> c7 [label="source back", color="#8FA875", fontcolor="#8FA875"]; - c2 -> c3 [label="still down", color="#BD8183", fontcolor="#BD8183"]; - c3 -> c4; - c4 -> c5 [label="available", color="#8FA875", fontcolor="#8FA875"]; - c4 -> c6 [label="58P01", color="#BD8183", fontcolor="#BD8183"]; - c5 -> c7; - } - - // Force A | B | C side by side via invisible head-to-head ordering - a0 -> b0 [style=invis, weight=100]; - b0 -> c0 [style=invis, weight=100]; - - manifest [label="manifest.toml\nsource identity + saved restart point\nrecent pipeline progress", fillcolor="#4D3850", shape=note]; - a9 -> manifest [style=invis]; - b8 -> manifest [style=invis]; - c7 -> manifest [style=invis]; - - // ═════ Legend ═════ - legend [shape=plaintext, label=< - - - - - - - - - - - -
node fill
operator / OS event
walshadow-stream
walshadow — segment / walsender side
shadow Postgres
ClickHouse (dedup endpoint)
scenario cluster border
━A. clean restart (SIGTERM)
━B. kill -9 mid-stream
━C. source → archive → source
- >]; - manifest -> legend [style=invis]; -} diff --git a/architecture/timeline_restart.svg b/architecture/timeline_restart.svg deleted file mode 100644 index 926d765a..00000000 --- a/architecture/timeline_restart.svg +++ /dev/null @@ -1,450 +0,0 @@ - - - - - - -timeline_restart - -walshadow restart timelines — 3 scenarios, time flows top → bottom within each column - -cluster_a - -A. clean restart  (SIGTERM, saved restart point still available) - - -cluster_b - -B. kill -9 mid-stream  (last saved manifest authoritative) - - -cluster_c - -C. restart recovery  (source → archive → source) - - - -a0 - -① SIGTERM - - - -a1 - -② save partial WAL segment - - - -a0->a1 - - - - - -b0 - -① kill -9 (no drain) - - - - -a2 - -③ drain queued work -and finish ClickHouse writes - - - -a1->a2 - - - - - -a3 - -④ latest manifest remains -conservative restart point - - - -a2->a3 - - - - - -a4 - -⑤ process exit - - - -a3->a4 - - - - - -a5 - -⑥ restart: -check source identity -load manifest.toml + toast_retires.toml -resume from saved point - - - -a4->a5 - - - - - -a6 - -⑦ Shadow::start in recovery mode -(if not already running) - - - -a5->a6 - - - - - -a7 - -⑧ walsender listener up -before walreceiver connects - - - -a6->a7 - - - - - -a8 - -⑨ resume source WAL stream -from saved point - - - -a7->a8 - - - - - -a9 - -⑩ records re-flow -CH ReplacingMergeTree(_lsn) dedups - - - -a8->a9 - - - - - -manifest - - - -manifest.toml -source identity + saved restart point -recent pipeline progress - - - - -b1 - -② OS reaps process -memory lost; last complete manifest survives -partial manifest write ignored - - - -b0->b1 - - - - - -c0 - -① restart: initialized shadow (no incomplete marker) -→ resume in place, never base backup -resume_lsn = manifest end_lsn (after bootstrap) -or --start-lsn - - - - -b2 - -③ restart: -clear transaction spill files -keep toast_retires.toml - - - -b1->b2 - - - - - -b3 - -④ check source identity -resume from saved restart point -flush safe mirror retirements - - - -b2->b3 - - - - - -b4 - -⑤ shadow PG recovers on its own -walreceiver retry timer - - - -b3->b4 - - - - - -b5 - -⑥ shadow asks START_REPLICATION -PHYSICAL <its flush_lsn> - - - -b4->b5 - - - - - -b6 - -⑦ resume source from saved point -resume shadow from its own progress - - - -b5->b6 - - - - - -b7 - -⑧ overlap records replayed -CH dedups on _lsn -shadow apply monotonic - - - -b6->b7 - - - - - -b8 - -⑨ steady-state reached - - - -b7->b8 - - - - - - -c1 - -② START_REPLICATION at resume_lsn -(stream.next_lsn) - - - -c0->c1 - - - - - -c2 - -③ transient drop → retry source once -(primary_conninfo first) - - - -c1->c2 - - -drop - - - -c3 - -④ fetch [backup] segment covering resume_lsn -(full segment sliced from resume_lsn); -replay through live filter + decode sinks - - - -c1->c3 - - -58P01 - - - -c7 - -⑦ steady-state reached -CH ReplacingMergeTree(_lsn) dedups - - - -c1->c7 - - -available - - - -c2->c3 - - -still down - - - -c2->c7 - - -source back - - - -c4 - -⑤ archive lacks next segment -reconnect source at exact handoff (backoff) - - - -c3->c4 - - - - - -c5 - -⑥ source has handoff WAL -resume live stream - - - -c4->c5 - - -available - - - -c6 - -⑥ 58P01 + archive can't cover -exit; operator decides base refresh - - - -c4->c6 - - -58P01 - - - -c5->c7 - - - - - - -legend - - - -node fill - - - -operator / OS event - - - -walshadow-stream - - - -walshadow — segment / walsender side - - - -shadow Postgres - - - -ClickHouse (dedup endpoint) - - -scenario cluster border - -━ - -A. clean restart (SIGTERM) - -━ - -B. kill -9 mid-stream - -━ - -C. source → archive → source - - - - diff --git a/architecture/timeline_streaming.dot b/architecture/timeline_streaming.dot deleted file mode 100644 index 62afe118..00000000 --- a/architecture/timeline_streaming.dot +++ /dev/null @@ -1,151 +0,0 @@ -// walshadow — steady-state streaming: one record's journey -// rankdir=TB — pipeline reads top→bottom; was LR but ran 6× wider than tall -digraph timeline_streaming { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow steady-state streaming — one record's journey through hot path", fontsize=14, splines=spline, nodesep=0.4, ranksep=0.6, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ═════ ① ingress ═════ - subgraph cluster_p1 { - label="① ingress"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - commit [label="COMMIT on demo.users\nUPDATE → WAL @ LSN=X", fillcolor="#3D3D54"]; - wsend [label="source walsender\nCopyData 'w' frame", fillcolor="#3D3D54"]; - feed [label="SourceFeed::pump\nWalChunk(start_lsn=X)", fillcolor="#4D3A28"]; - commit -> wsend -> feed [color="#A1A9CC"]; - } - - // ═════ ② filter + rewrite ═════ - subgraph cluster_p2 { - label="② filter + rewrite"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - push [label="WalStream::push\nStreamingWalker\nextend / try_next", fillcolor="#4D3A28"]; - rec [label="CompletedRecord\n(parsed, byte_ranges)", fillcolor="#4D3A28", shape=parallelogram]; - decide [label="Filter::decide\n→ Decision::Drop\n(user heap)", fillcolor="#4D3A28"]; - rewr [label="rewrite::noop_replace\nin-place + CRC32C", fillcolor="#4D3A28"]; - push -> rec -> decide -> rewr; - } - feed -> push [lhead=cluster_p2]; - - // ═════ ③ fan-out (pump task — synchronous dispatch in order) ═════ - subgraph cluster_p3 { - label="③ fan-out (pump task, sync — ordering matters)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - fbytes [label="❶ on_record_bytes\n→ ShadowStreamSink\n(stays on pump)", fillcolor="#4D3340"]; - frec [label="❷ on_record →\nQueueingRecordSink\n(clone + enqueue,\n returns immediately)", fillcolor="#4D3340"]; - fseg [label="❸ on_segment defers\nuntil 16 MiB boundary", fillcolor="#4D3340"]; - } - rewr -> fbytes [lhead=cluster_p3]; - rewr -> frec; - rewr -> fseg [style=dashed]; - - // ═════ ④ shadow apply (hot wire — independent of decoder) ═════ - subgraph cluster_p4 { - label="④ shadow apply (record cadence, ms — pump task)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - frame4 [label="frame 'w' XLogData\nenqueue to send buffer", fillcolor="#4D3340"]; - recv4 [label="walreceiver applies redo\npg_last_wal_replay_lsn ↑", fillcolor="#3D4128"]; - frame4 -> recv4 [label="hot wire", color="#BD8183", penwidth=2]; - } - fbytes -> frame4 [lhead=cluster_p4, color="#BD8183", penwidth=2]; - - // ═════ ④' QueueingRecordSink hand-off ═════ - subgraph cluster_pQ { - label="④' QueueingRecordSink hand-off (pump → worker, decouples ④ from ⑤)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - qbuf [label="pump-side batch\nbatch_size = 64", fillcolor="#4D3A28", shape=parallelogram]; - qchn [label="mpsc batch send\n(soft cap → yield)", fillcolor="#4D3A28", shape=parallelogram]; - qwrk [label="worker task picks up\non_record / on_idle\nticks", fillcolor="#4D3A28"]; - qbuf -> qchn -> qwrk; - } - frec -> qbuf [lhead=cluster_pQ]; - - // ═════ ⑤ decoder gate + xact buffer (worker task) ═════ - subgraph cluster_p5 { - label="⑤ decoder gate clears + xact buffer (worker task — gated on ④)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - gate5 [label="BufferingDecoderSink\nrelation_at(rfn, X)\nwait_for_replay(X)", fillcolor="#4D4128"]; - dec5 [label="heap_decoder\n→ DecodedHeap", fillcolor="#4D4128"]; - buf5 [label="XactBuffer\nHeap / Chunk / ToastDelete / Raw\nper-xid mem + spill v4\nSMGR markers + SubxactTracker", fillcolor="#4D4128"]; - gate5 -> dec5 -> buf5; - } - qwrk -> gate5 [lhead=cluster_p5]; - recv4 -> gate5 [label="apply_lsn ≥ X\n(unblocks gate)", style=dashed, color="#BD8183"]; - - // ═════ ⑥ commit drain → CH (parallel pipeline) ═════ - subgraph cluster_p6 { - label="⑥ XLOG_XACT_COMMIT → reorder → decode → batcher → inserter (pipeline)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - cmt6 [label="XLOG_XACT_COMMIT\nLSN = X'", fillcolor="#3D3D54"]; - drain6 [label="transaction reorder\nwalk rows + changes in WAL order\nsave TOAST changes before publishing commit", fillcolor="#4D4128"]; - dec6 [label="decode worker (×M pool)\ndetoast via ChunkMap / as-of mirror fetch\nresolve + mapping\nroute main rows; Placed(seq, rows)", fillcolor="#4D4128"]; - enc6 [label="InsertBatcher\nTableEncoder::append_row\n_lsn / _xid / _commit_ts / _is_deleted\nbuffer across xacts", fillcolor="#4D4128"]; - seal6 [label="seal InsertBatch\nbudget / per-table deadline\n(owned slabs + per_seq)", fillcolor="#4D4128"]; - close6 [label="inserter (×N pool)\none complete INSERT\nsend_query → send_data → EndOfStream\nAcked(per_seq) after drain", fillcolor="#4D4128"]; - ack6 [label="ack collector\npublish X' after every\nearlier commit completes", fillcolor="#4D3A28", shape=note]; - ch6 [label="ClickHouse\nmain ReplacingMergeTree(_lsn)\nTID-keyed pg_toast_ mirrors", fillcolor="#4D4128"]; - - cmt6 -> drain6; - drain6 -> dec6 [label="DecodeJob"]; - dec6 -> enc6; - enc6 -> seal6 [style=dashed, label="budget / deadline"]; - seal6 -> close6; - close6 -> ch6 [label="Native block", color="#BF8C5F"]; - close6 -> ack6 [style=dotted, color="#b380b0"]; - } - buf5 -> cmt6 [label="next record", style=dashed]; - - // ═════ ⑦ async durability (off hot path) ═════ - subgraph cluster_p7 { - label="⑦ async durability (off hot path, segment / manifest cadence)"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - seg7 [label="DirSegmentSink::on_segment\nfsync segment + manifest\nfsync parent dir", fillcolor="#4D3340"]; - out7 [label="out/\n+ manifest.json", fillcolor="#4D3850", shape=note]; - status7 [label="status loop\nchoose safe restart point\nsave manifest, then publish point\nand send source feedback", fillcolor="#4D3A28"]; - manifest7 [label="manifest.toml\nsource identity + restart state", fillcolor="#4D3850", shape=note]; - slot7 [label="source slot advance\nnever past shadow replay\nor durable ClickHouse data", fillcolor="#3D3D54"]; - - seg7 -> out7; - seg7 -> status7 [style=dotted, color="#b380b0", label="saved WAL"]; - status7 -> manifest7 [style=dotted, color="#b380b0", penwidth=2, label="save first"]; - status7 -> slot7 [color="#B58B86", style=dashed]; - } - fseg -> seg7 [label="16 MiB boundary", style=dashed, lhead=cluster_p7]; - ack6 -> status7 [style=dotted, color="#b380b0", label="durable progress"]; - - // ═════ Legend + budget table ═════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - - - -
node fill — actor
source Postgres
walshadow ingress / queue / orchestrator
walshadow decoder + xact + emitter / CH
walshadow output sinks (walsender, segment)
shadow Postgres
on-disk artifact
edge colour
━━source → walshadow replication
━━walsender wire → shadow (hot path)
━━ClickHouse Native block
┄┄standby status to source slot
···ack + manifest durability
hot-path budget
④ shadow applyms — walsender wire (independent of ⑤)
④' queue hand-off~ns enqueue, batch_size=64
⑤ gate clearms — driven by ④ apply_lsn
⑥ CH pipelinerows buffered across xacts; budget/deadline seals one complete INSERT per batch; N INSERTs in flight; ack on contiguous-done
⑦ segment + manifestseconds — async, off hot path; save restart point before sharing it
- >]; - ch6 -> legend [style=invis]; -} diff --git a/architecture/timeline_streaming.svg b/architecture/timeline_streaming.svg deleted file mode 100644 index 42be5358..00000000 --- a/architecture/timeline_streaming.svg +++ /dev/null @@ -1,582 +0,0 @@ - - - - - - -timeline_streaming - -walshadow steady-state streaming — one record's journey through hot path - -cluster_p1 - -① ingress - - -cluster_p2 - -② filter + rewrite - - -cluster_p3 - -③ fan-out (pump task, sync — ordering matters) - - -cluster_p4 - -④ shadow apply  (record cadence, ms — pump task) - - -cluster_pQ - -④' QueueingRecordSink hand-off  (pump → worker, decouples ④ from ⑤) - - -cluster_p5 - -⑤ decoder gate clears + xact buffer  (worker task — gated on ④) - - -cluster_p6 - -⑥ XLOG_XACT_COMMIT  → reorder → decode → batcher → inserter (pipeline) - - -cluster_p7 - -⑦ async durability  (off hot path, segment / manifest cadence) - - - -commit - -COMMIT on demo.users -UPDATE → WAL @ LSN=X - - - -wsend - -source walsender -CopyData 'w' frame - - - -commit->wsend - - - - - -feed - -SourceFeed::pump -WalChunk(start_lsn=X) - - - -wsend->feed - - - - - -push - -WalStream::push -StreamingWalker -extend / try_next - - - -feed->push - - - - - -rec - -CompletedRecord -(parsed, byte_ranges) - - - -push->rec - - - - - -decide - -Filter::decide -→ Decision::Drop -(user heap) - - - -rec->decide - - - - - -rewr - -rewrite::noop_replace -in-place + CRC32C - - - -decide->rewr - - - - - -fbytes - -❶ on_record_bytes -→ ShadowStreamSink -(stays on pump) - - - -rewr->fbytes - - - - - -frec - -❷ on_record → -QueueingRecordSink -(clone + enqueue, - returns immediately) - - - -rewr->frec - - - - - -fseg - -❸ on_segment defers -until 16 MiB boundary - - - -rewr->fseg - - - - - -frame4 - -frame 'w' XLogData -enqueue to send buffer - - - -fbytes->frame4 - - - - - -qbuf - -pump-side batch -batch_size = 64 - - - -frec->qbuf - - - - - -seg7 - -DirSegmentSink::on_segment -fsync segment + manifest -fsync parent dir - - - -fseg->seg7 - - -16 MiB boundary - - - -recv4 - -walreceiver applies redo -pg_last_wal_replay_lsn ↑ - - - -frame4->recv4 - - -hot wire - - - -gate5 - -BufferingDecoderSink -relation_at(rfn, X) -wait_for_replay(X) - - - -recv4->gate5 - - -apply_lsn ≥ X -(unblocks gate) - - - -qchn - -mpsc batch send -(soft cap → yield) - - - -qbuf->qchn - - - - - -qwrk - -worker task picks up -on_record / on_idle -ticks - - - -qchn->qwrk - - - - - -qwrk->gate5 - - - - - -dec5 - -heap_decoder -→ DecodedHeap - - - -gate5->dec5 - - - - - -buf5 - -XactBuffer -Heap / Chunk / ToastDelete / Raw -per-xid mem + spill v4 -SMGR markers + SubxactTracker - - - -dec5->buf5 - - - - - -cmt6 - -XLOG_XACT_COMMIT -LSN = X' - - - -buf5->cmt6 - - -next record - - - -drain6 - -transaction reorder -walk rows + changes in WAL order -save TOAST changes before publishing commit - - - -cmt6->drain6 - - - - - -dec6 - -decode worker (×M pool) -detoast via ChunkMap / as-of mirror fetch -resolve + mapping -route main rows; Placed(seq, rows) - - - -drain6->dec6 - - -DecodeJob - - - -enc6 - -InsertBatcher -TableEncoder::append_row -_lsn / _xid / _commit_ts / _is_deleted -buffer across xacts - - - -dec6->enc6 - - - - - -seal6 - -seal InsertBatch -budget / per-table deadline -(owned slabs + per_seq) - - - -enc6->seal6 - - -budget / deadline - - - -close6 - -inserter (×N pool) -one complete INSERT -send_query → send_data → EndOfStream -Acked(per_seq) after drain - - - -seal6->close6 - - - - - -ack6 - - - -ack collector -publish X' after every -earlier commit completes - - - -close6->ack6 - - - - - -ch6 - -ClickHouse -main ReplacingMergeTree(_lsn) -TID-keyed pg_toast_<relid> mirrors - - - -close6->ch6 - - -Native block - - - -status7 - -status loop -choose safe restart point -save manifest, then publish point -and send source feedback - - - -ack6->status7 - - -durable progress - - - -legend - - - -node fill — actor - - - -source Postgres - - - -walshadow ingress / queue / orchestrator - - - -walshadow decoder + xact + emitter / CH - - - -walshadow output sinks (walsender, segment) - - - -shadow Postgres - - - -on-disk artifact - - -edge colour - -━━ - -source → walshadow replication - -━━ - -walsender wire → shadow (hot path) - -━━ - -ClickHouse Native block - -┄┄ - -standby status to source slot - -··· - -ack + manifest durability - - -hot-path budget - -④ shadow apply - -ms — walsender wire (independent of ⑤) - -④' queue hand-off - -~ns enqueue, batch_size=64 - -⑤ gate clear - -ms — driven by ④ apply_lsn - -⑥ CH pipeline - -rows buffered across xacts; budget/deadline seals one complete INSERT per batch; N INSERTs in flight; ack on contiguous-done - -⑦ segment + manifest - -seconds — async, off hot path; save restart point before sharing it - - - - -out7 - - - -out/<seg> -+ manifest.json - - - -seg7->out7 - - - - - -seg7->status7 - - -saved WAL - - - -manifest7 - - - -manifest.toml -source identity + restart state - - - -status7->manifest7 - - -save first - - - -slot7 - -source slot advance -never past shadow replay -or durable ClickHouse data - - - -status7->slot7 - - - - - diff --git a/architecture/toast.dot b/architecture/toast.dot deleted file mode 100644 index 1d12e127..00000000 --- a/architecture/toast.dot +++ /dev/null @@ -1,108 +0,0 @@ -// walshadow — TOAST resolution, ClickHouse mirror, and lifecycle -// -// regeneration spec: -// sources of truth: plans/TOAST.md · src/toast/{resolver,toast_retire}.rs · src/xact/xact_buffer.rs · src/emit/pipeline/{bootstrap,reorder,decode}.rs -// subsumes: plans/TOAST.md § Store / Lifecycle / Scope limits -// quality bar: -// - in-xact fast path and store-backed slow path stay distinct -// - births/tombstones reach mirror before publishing ack -// - TRUNCATE, DROP retirement, and rewrite barrier show different semantics -// - bootstrap put-before-resolve ordering remains visible -// shared style: palette.md -digraph toast { - rankdir=TB; - compound=true; - newrank=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow TOAST — in-xact reassembly + TID-keyed ClickHouse mirror + lifecycle", fontsize=14, splines=spline, nodesep=0.4, ranksep=0.6, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, arrowsize=0.8, color="#c1a78e", fontcolor="#ECE1D7"]; - - wal [label="source WAL\nheap + pg_toast records", fillcolor="#3D3D54", shape=parallelogram]; - - subgraph cluster_stream { - label="streaming commit — WAL order, abort-safe"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - decode [label="BufferingDecoderSink\nmain heap → DecodedHeap\ntoast INSERT → ToastChunk\ntoast DELETE → ToastDelete\nTID + record LSN", fillcolor="#4D4128"]; - spill [label="XactBuffer / spill v4\nHeap | Chunk | ToastDelete | Raw\nabort discards", fillcolor="#4D4128"]; - drain [label="committed transaction\nshared ordered walk keeps chunks,\nrows, and table changes together", fillcolor="#4D4128"]; - detoast [label="decode pool\ndetoast_heap\ntry in-xact ChunkMap first", fillcolor="#5D4628"]; - rows [label="main-table rows\ninline Text / Bytea / PgPending", fillcolor="#5D4628", shape=parallelogram]; - - decode -> spill -> drain; - drain -> detoast [label="heaps + chunk maps"]; - detoast -> rows; - } - wal -> decode; - - subgraph cluster_store { - label="ToastResolver / ChunkStore — store-backed slow path"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - resolver [label="ToastResolver\ndisabled | clickhouse", fillcolor="#4D4D28"]; - put [label="put(new_rows)\nbefore commit publishing marker", fillcolor="#4D4D28"]; - fetch [label="fetch(relid, value_id, max_lsn)\nper-TID latest ≤ max_lsn\nlive rows → newest per seq", fillcolor="#4D4D28"]; - mirror [label="CH pg_toast_\nReplacingMergeTree(_lsn, _is_deleted)\nORDER BY (blkno, offnum)\nbirth or tombstone per TID", fillcolor="#4D4128", shape=cylinder]; - miss [label="miss policy\ndisabled → default fill\nstore miss → superseded fill\nshort run → mismatch fill\nmissing mirror → fatal", fillcolor="#5D3F40", shape=note]; - - resolver -> put; - resolver -> fetch; - put -> mirror [color="#BF8C5F", penwidth=2]; - fetch -> mirror [dir=both, arrowtail=open, color="#CBA85E"]; - fetch -> miss [style=dashed]; - } - drain -> put [label="new_rows\nWAL ordered", color="#BF8C5F"]; - detoast -> fetch [label="pointer absent from\nin-xact maps", color="#CBA85E", style=dashed]; - - subgraph cluster_bootstrap { - label="bootstrap — put all chunks before deferred resolution"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - walk [label="PageWalkSink\nmain tuples + pg_toast tuples\non-page TID, walk LSN", fillcolor="#4D3340"]; - defer [label="pipeline::bootstrap::drain\nbatch ToastRows\ndefer mapped ExternalToast", fillcolor="#4D3340"]; - resolve [label="flush chunks → resolve deferred\nbootstrap miss = hard error", fillcolor="#4D3340"]; - walk -> defer -> resolve; - } - defer -> put [label="ToastRows", color="#BF8C5F"]; - resolve -> fetch [label="after flush", color="#CBA85E", style=dashed]; - resolve -> rows [label="resolved bootstrap rows", color="#BF8C5F"]; - - subgraph cluster_lifecycle { - label="lifecycle barriers — TRUNCATE, DROP, rewrite stay distinct"; - style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - trunc [label="owner TRUNCATE\nfence main rows\nTRUNCATE dest + mirror in slice", fillcolor="#5D4628"]; - drop [label="toast SchemaEvent::Dropped\nenqueue (relid, commit_lsn)", fillcolor="#5D4628"]; - ledger [label="toast_retires.toml\nsaved retirements", fillcolor="#4D3850", shape=note]; - floor [label="saved restart point\nhas passed drop", fillcolor="#4D3A28"]; - retire [label="retire_mirror\nTRUNCATE table, never DROP", fillcolor="#5D4628"]; - marker [label="XLOG_SMGR_CREATE marker\n+ SpillEntry::Raw stash\nresolve at commit", fillcolor="#5D4628"]; - barrier [label="rewrite_barrier O − B\nresidual tombstones @ commit LSN\nafter generation births", fillcolor="#5D4628"]; - - drop -> ledger [color="#6E6963", style=dashed]; - ledger -> floor [style=dashed]; - floor -> retire; - marker -> barrier; - } - trunc -> mirror [label="wipe", color="#BF8C5F", style=dashed]; - retire -> mirror [label="wipe when replay-safe", color="#BF8C5F", style=dashed]; - barrier -> mirror [label="insert residual deaths", color="#BF8C5F", style=dashed]; - drain -> trunc [label="truncate step", style=dashed]; - drain -> drop [label="drop step", style=dashed]; - decode -> marker [label="invisible generation", style=dashed]; - - main [label="CH destination tables\nReplacingMergeTree(_lsn)", fillcolor="#4D4128", shape=cylinder]; - rows -> main [color="#BF8C5F", penwidth=2]; - - legend [shape=plaintext, label=< - - - - - - - -
invariants
same-xactChunkMap wins, no store read
store writebirths/tombstones durable before commit publication
as-of readfuture TID generations invisible at referring LSN
DROPpersist intent before ack, wipe after replay floor passes
rewritepreserve old as-of windows, close residuals with O − B
- >]; - main -> legend [style=invis]; -} diff --git a/architecture/toast.svg b/architecture/toast.svg deleted file mode 100644 index dde5eeb4..00000000 --- a/architecture/toast.svg +++ /dev/null @@ -1,423 +0,0 @@ - - - - - - -toast - -walshadow TOAST — in-xact reassembly + TID-keyed ClickHouse mirror + lifecycle - -cluster_stream - -streaming commit — WAL order, abort-safe - - -cluster_store - -ToastResolver / ChunkStore — store-backed slow path - - -cluster_bootstrap - -bootstrap — put all chunks before deferred resolution - - -cluster_lifecycle - -lifecycle barriers — TRUNCATE, DROP, rewrite stay distinct - - - -wal - -source WAL -heap + pg_toast records - - - -decode - -BufferingDecoderSink -main heap → DecodedHeap -toast INSERT → ToastChunk -toast DELETE → ToastDelete -TID + record LSN - - - -wal->decode - - - - - -spill - -XactBuffer / spill v4 -Heap | Chunk | ToastDelete | Raw -abort discards - - - -decode->spill - - - - - -marker - -XLOG_SMGR_CREATE marker -+ SpillEntry::Raw stash -resolve at commit - - - -decode->marker - - -invisible generation - - - -drain - -committed transaction -shared ordered walk keeps chunks, -rows, and table changes together - - - -spill->drain - - - - - -detoast - -decode pool -detoast_heap -try in-xact ChunkMap first - - - -drain->detoast - - -heaps + chunk maps - - - -put - -put(new_rows) -before commit publishing marker - - - -drain->put - - -new_rows -WAL ordered - - - -trunc - -owner TRUNCATE -fence main rows -TRUNCATE dest + mirror in slice - - - -drain->trunc - - -truncate step - - - -drop - -toast SchemaEvent::Dropped -enqueue (relid, commit_lsn) - - - -drain->drop - - -drop step - - - -rows - -main-table rows -inline Text / Bytea / PgPending - - - -detoast->rows - - - - - -fetch - -fetch(relid, value_id, max_lsn) -per-TID latest ≤ max_lsn -live rows → newest per seq - - - -detoast->fetch - - -pointer absent from -in-xact maps - - - -main - - -CH destination tables -ReplacingMergeTree(_lsn) - - - -rows->main - - - - - -resolver - -ToastResolver -disabled | clickhouse - - - -resolver->put - - - - - -resolver->fetch - - - - - -mirror - - -CH pg_toast_<relid> -ReplacingMergeTree(_lsn, _is_deleted) -ORDER BY (blkno, offnum) -birth or tombstone per TID - - - -put->mirror - - - - - -fetch->mirror - - - - - - -miss - - - -miss policy -disabled → default fill -store miss → superseded fill -short run → mismatch fill -missing mirror → fatal - - - -fetch->miss - - - - - -walk - -PageWalkSink -main tuples + pg_toast tuples -on-page TID, walk LSN - - - -defer - -pipeline::bootstrap::drain -batch ToastRows -defer mapped ExternalToast - - - -walk->defer - - - - - -defer->put - - -ToastRows - - - -resolve - -flush chunks → resolve deferred -bootstrap miss = hard error - - - -defer->resolve - - - - - -resolve->rows - - -resolved bootstrap rows - - - -resolve->fetch - - -after flush - - - -trunc->mirror - - -wipe - - - -ledger - - - -toast_retires.toml -saved retirements - - - -drop->ledger - - - - - -floor - -saved restart point -has passed drop - - - -ledger->floor - - - - - -retire - -retire_mirror -TRUNCATE table, never DROP - - - -floor->retire - - - - - -retire->mirror - - -wipe when replay-safe - - - -barrier - -rewrite_barrier O − B -residual tombstones @ commit LSN -after generation births - - - -marker->barrier - - - - - -barrier->mirror - - -insert residual deaths - - - -legend - - - -invariants - -same-xact - -ChunkMap wins, no store read - -store write - -births/tombstones durable before commit publication - -as-of read - -future TID generations invisible at referring LSN - -DROP - -persist intent before ack, wipe after replay floor passes - -rewrite - -preserve old as-of windows, close residuals with O − B - - - - diff --git a/architecture/values.svg b/architecture/values.svg new file mode 100644 index 00000000..c5094481 --- /dev/null +++ b/architecture/values.svg @@ -0,0 +1,71 @@ + + TOAST resolution and PostgreSQL type conversion + Transaction heaps and TOAST chunks enter planning and resolution. ToastResolver reads or writes versioned ClickHouse mirrors for values outside the current transaction. InsertBatcher stores locally encoded column slabs or Oracle cells. Inserter sends one ENCODE_NATIVE request per batch to shadow, validates returned Native columns, combines them with local slabs and sends the complete ClickHouse INSERT. + + + + + + + + + XactBuffer + Heap tuples + TOAST chunks + In-memory or spilled + + ReorderSink / ToastResolver + Plan + resolve external values + Persist TOAST before dispatch + + pg_toast_<relid> + ClickHouse chunk history + Births / tombstones by LSN + + + put/fetch + + DecodeJob → decoded RowChunk + + InsertBatcher / TableEncoder + ColumnEncoding selects local slabs or Oracle cells; both remain in one sealed InsertBatch + + + + ColumnBuf + Local encoded columns + + ColumnBuf::Oracle + Raw Datum / text / default + Inserter calls encode_batch + + Shadow PostgreSQL + + walshadow worker + Bridge: ENCODE_NATIVE + One request per batch + Return partial Native block + + + Native response checks + Validate row count, names, + column types and block shape + + Native columns + + BlockBuilder + Local + returned columns + Retain block across retries + + + + + ClickHouse + One complete INSERT + Conversion or validation failure stops batch before INSERT + No Acked event; restart can replay unfinished work + diff --git a/architecture/workers.svg b/architecture/workers.svg new file mode 100644 index 00000000..cd404c50 --- /dev/null +++ b/architecture/workers.svg @@ -0,0 +1,105 @@ + + Commit pipeline, queues and acknowledgements + One ReorderSink assigns sequence numbers and enqueues slices. A bounded shared DecodeJob queue fans out to M decode tasks. Rows merge into one FIFO BatcherMsg channel and InsertBatcher. A shared InsertBatch queue fans out to N inserters. Register, Placed and Acked events feed AckCollector independently of row flow; only contiguous completed work advances emitter_ack_lsn. + + + + + + + + + ReorderSink · record-queue worker + Plan committed transaction, attach descriptor + route, assign seq + One commit may produce several DecodeJob slices + + AckCollector · 1 task + + Register + seq → commit_lsn + Track each work slice + Final slice publishes LSN + + DecodeJob · bounded shared queue + + + + + + + decode pool · M tasks + + decode[1] + + decode[2] + + decode[M] + + + + + + + Placed + seq → row count + After all rows enter + BatcherMsg channel + + + + + + + BatcherMsg::Rows / FlushAll · FIFO mpsc + + + + + + InsertBatcher · 1 task + TableEncoder per table; seal on rows, bytes, deadline, or FlushAll + + + InsertBatch · shared queue, any idle inserter + + + + + + inserter pool · N tasks + + inserter[1] + + inserter[2] + + inserter[N] + + + + + + + Acked + seq → inserted rows + Only after ClickHouse + returns EndOfStream + + + + + + + ClickHouse · N connections, complete Native INSERTs + + Advance through contiguous + completed seqs only + emitter_ack_lsn + + Status loop → saved restart floor + M = decoder_pool_size N = inserter_pool_size + diff --git a/architecture/xact.dot b/architecture/xact.dot deleted file mode 100644 index 6d16d9d8..00000000 --- a/architecture/xact.dot +++ /dev/null @@ -1,126 +0,0 @@ -// walshadow — XactBuffer lifecycle (insert → maybe-evict → commit drain) -// Strict layered DAG: producers above their consumers, single cluster -// for the drain pipeline, all other nodes bare. No constraint=false -// except the abort→spill backwards unlink. -// -// regeneration spec: -// sources of truth: plans/xact.md · src/xact/{xact_buffer,spill}.rs · src/emit/pipeline/reorder.rs · src/backfill/backup_backfill.rs -// subsumes: plans/xact.md § "Buffer shape" + "Eviction policy" + "Spill backend" + "Drain shape" -// quality bar: -// - spill round-trip (write → file → read) visually traceable -// - subx tracker feeds drain without overlapping spill IO -// - evbus → kmerge edge doesn't entangle with main merge edges -// shared style: palette.md -digraph xact { - rankdir=TB; - compound=true; - graph [fontname="Helvetica", labelloc="t", label="walshadow — XactBuffer + spill + commit drain", fontsize=14, splines=spline, nodesep=0.35, ranksep=0.55, bgcolor="#272623", fontcolor="#ECE1D7"]; - node [fontname="Helvetica", fontsize=10, shape=box, style="rounded,filled", color="#6E6963", fontcolor="#ECE1D7"]; - edge [fontname="Helvetica", fontsize=9, color="#c1a78e", fontcolor="#ECE1D7"]; - - // ════════ Rank 0: producers ════════ - // catevt+decoder on the left (both feed buf); xaclog on the right - // (feeds subt + abort + merge, all aligned beneath it) - { rank=same; - catevt [label="ShadowCatalog\nschema_event_tx\nAdded / Changed / Dropped", fillcolor="#4D4D28"]; - decoder [label="BufferingDecoderSink\nheap + TOAST birth/death reshape\nraw stash: dirty tree · SMGR marker ·\nspanned-lookup miss (NotCovered /\nDropped / Ambiguous)\nstamps source_lsn, xid", fillcolor="#4D4128"]; - xaclog [label="XLOG_XACT_*\nASSIGNMENT 0x50 (hint)\nCOMMIT 0x00 / _PREPARED 0x30\nABORT 0x20 / _PREPARED 0x40", fillcolor="#3D3D54"]; - } - - // ════════ Rank 1: state — buf | subt | abort | idle, in producer-aligned order ════════ - { rank=same; - buf [label=< - - - - - - - - - - -
XactBuffer
inflight: HashMap<xid, XactState>
markers · pending stash · unfinished commits
bytes_in_memory · drain_resident · SpillStore
XactState (per xid):
first_lsn · in_mem: Vec<SpillEntry> · in_mem_bytes
spill: Option<SpillWriter> · spill_bytes
events: Vec<(lsn, DrainEntry)> · stash_rfns
SpillEntry = Heap | Chunk | ToastDelete | Raw
- >, fillcolor="#4D4128", shape=box]; - - subt [label="SubxactTracker\nparent: HashMap\nchildren: HashMap>", fillcolor="#4D4128"]; - - abort [label="abort\ndrop transaction + children\nunlink spill files\nadvance consumed position", fillcolor="#4D4128"]; - - idle [label="idle progress\nadvance only with no\nopen transaction", fillcolor="#4D4128"]; - } - - // producer → state (vertical drops, no cross-traffic) - catevt -> buf [label="on_schema_event\n(xid, source_lsn, ev)", color="#CBA85E", style=dashed]; - decoder -> buf [label="absorb Heap / Chunk /\nToastDelete / Raw"]; - xaclog -> subt [label="ASSIGNMENT 0x50\nadd_subxact(top, subs)"]; - xaclog -> abort [label="ABORT / _PREPARED"]; - buf -> idle [style=dashed, label="xacts_active == 0"]; - - // ════════ Rank 2: evict + spill (sidecar pair under buf) ════════ - { rank=same; - evict [label="maybe_evict\nbytes_in_memory > xact_buffer_max (64 MiB)\npick largest in-mem xact\n(mirrors PG ReorderBufferLargestTXN)\n→ evict_xact: lazy-open SpillWriter,\ndrain in_mem → write(entry), zero bytes\nDrainEntry events stay in memory", fillcolor="#4D3A28"]; - spill [label="{spill}/xid-{xid:010}-{first_lsn:016X}.bin\n[\"WS\" magic | u16 ver=6]\n[tag u8 | u32 LE inner_len | body]*\n0 Heap · 1 Chunk · 2 ToastDelete ·\n3 Raw · 4 descriptor dict\nappend-only, fsync on finish", fillcolor="#4D3850", shape=note]; - } - buf -> evict [label="after every absorb\nwhile over budget"]; - evict -> spill [color="#6E6963", style=dashed, label="SpillWriter::write"]; - abort -> spill [color="#6E6963", style=dashed, label="unlink"]; - - // ════════ Rank 3+: commit drain (linear pipeline, the only remaining cluster) ════════ - subgraph cluster_drain { - label="commit drain — merge transaction family, preserve WAL order"; style="rounded,filled"; color="#4c4641"; fillcolor="#34302c"; fontcolor="#ECE1D7"; - - merge [label="pull (top + subxids) from inflight\nper xid: SpillReader + in_mem + events\n→ source_lsn k-way merge\nChunk → ChunkMap + ToastRow birth\nToastDelete → tombstone row\nRaw → resolve_stash verdict at commit next_lsn:\n Toast → chunk decode · Ordinary → row fanout\n Ambiguous → fatal · tombstoned/uncovered → discard\nTIE: control event BEFORE heap", fillcolor="#4D4128"]; - - dispatch [label="bounded committed batches\none shared walk keeps rows, TOAST,\nand table changes in WAL order", fillcolor="#4D4128"]; - - merge -> dispatch; - } - xaclog -> merge [label="COMMIT / _PREPARED\nparse_xact_payload"]; - subt -> merge [label="(top + subxids)\nauthoritative on COMMIT", style=dashed]; - spill -> merge [color="#6E6963", style=dashed, label="SpillReader::next\nthen unlink"]; - - // ════════ Rank below drain: shared consumers ════════ - detoast [label="resolve large values\ncurrent transaction first,\nmirror history second", fillcolor="#4D4128"]; - consumer [label="shared consumers\nlive pipeline: ClickHouse or metrics only\nbackup replay: same ordered walk", fillcolor="#5D4628"]; - dispatch -> detoast [color="#BF8C5F", penwidth=2, label="rows"]; - detoast -> consumer [color="#BF8C5F", penwidth=2, label="decoded rows"]; - dispatch -> consumer [color="#BF8C5F", style=dashed, label="table changes"]; - store [label="TOAST mirror\nsave changes before publishing commit\ninterleave table wipes and rewrites", fillcolor="#5D4628", shape=cylinder]; - dispatch -> store [color="#BF8C5F", style=dashed, label="TOAST changes"]; - - // ════════ Bottom: durable floor fan-in ════════ - safe [label="ack collector + transaction state\nchoose safe restart point\nstay behind open or unfinished work", fillcolor="#4D3A28"]; - manifest [label="manifest.toml\nsaved restart state", fillcolor="#4D3850", shape=note]; - consumer -> safe [color="#b380b0", style=dotted, label="downstream completed"]; - buf -> safe [color="#b380b0", style=dotted, label="open transactions"]; - dispatch -> safe [color="#b380b0", style=dotted, label="drained"]; - abort -> safe [color="#b380b0", style=dotted]; - idle -> safe [color="#b380b0", style=dotted]; - safe -> manifest [color="#b380b0", style=dotted, penwidth=2, label="status update"]; - - // ════════ Legend ════════ - legend [shape=plaintext, label=< - - - - - - - - - - - - - - - - - - - -
node fill — role
source PG / XLOG_XACT records
eviction policy (sync, after absorb)
XactBuffer / decoder / drain
ShadowCatalog schema_event_tx
downstream consumer (pipeline / gap replay)
on-disk artifact (spill / manifest)
edge colour
━━buffer-internal control / data
┄┄schema event (lsn-stamped)
━━walk steps to pipeline / replay consumers
┄┄spill file IO (write / read / unlink)
···ack + manifest durability
k-way merge — ordering rules
heads sorted ASC by source_lsn
tie: catalog event BEFORE tuple (PG writes pg_class first)
heaps detoast post-merge; chunk maps resolve rows; ToastRows persist separately
saved restart point never passes open or unfinished transaction
- >]; - manifest -> legend [style=invis]; -} diff --git a/architecture/xact.svg b/architecture/xact.svg deleted file mode 100644 index 1267f5b6..00000000 --- a/architecture/xact.svg +++ /dev/null @@ -1,400 +0,0 @@ - - - - - - -xact - -walshadow — XactBuffer + spill + commit drain - -cluster_drain - -commit drain — merge transaction family, preserve WAL order - - - -catevt - -ShadowCatalog -schema_event_tx -Added / Changed / Dropped - - - -buf - -XactBuffer -inflight: HashMap<xid, XactState> -markers · pending stash · unfinished commits -bytes_in_memory · drain_resident · SpillStore -XactState (per xid): -first_lsn · in_mem: Vec<SpillEntry> · in_mem_bytes -spill: Option<SpillWriter> · spill_bytes -events: Vec<(lsn, DrainEntry)> · stash_rfns -SpillEntry = Heap | Chunk | ToastDelete | Raw - - - -catevt->buf - - -on_schema_event -(xid, source_lsn, ev) - - - -decoder - -BufferingDecoderSink -heap + TOAST birth/death reshape -raw stash: dirty tree · SMGR marker · -spanned-lookup miss (NotCovered / -Dropped / Ambiguous) -stamps source_lsn, xid - - - -decoder->buf - - -absorb Heap / Chunk / -ToastDelete / Raw - - - -xaclog - -XLOG_XACT_* -ASSIGNMENT 0x50 (hint) -COMMIT 0x00 / _PREPARED 0x30 -ABORT  0x20 / _PREPARED  0x40 - - - -subt - -SubxactTracker -parent: HashMap<xid, top> -children: HashMap<top, Vec<xid>> - - - -xaclog->subt - - -ASSIGNMENT 0x50 -add_subxact(top, subs) - - - -abort - -abort -drop transaction + children -unlink spill files -advance consumed position - - - -xaclog->abort - - -ABORT / _PREPARED - - - -merge - -pull (top + subxids) from inflight -per xid: SpillReader + in_mem + events -→ source_lsn k-way merge -Chunk → ChunkMap + ToastRow birth -ToastDelete → tombstone row -Raw → resolve_stash verdict at commit next_lsn: -  Toast → chunk decode · Ordinary → row fanout -  Ambiguous → fatal · tombstoned/uncovered → discard -TIE: control event BEFORE heap - - - -xaclog->merge - - -COMMIT / _PREPARED -parse_xact_payload - - - -idle - -idle progress -advance only with no -open transaction - - - -buf->idle - - -xacts_active == 0 - - - -evict - -maybe_evict -bytes_in_memory > xact_buffer_max (64 MiB) -pick largest in-mem xact -(mirrors PG ReorderBufferLargestTXN) -→ evict_xact: lazy-open SpillWriter, -drain in_mem → write(entry), zero bytes -DrainEntry events stay in memory - - - -buf->evict - - -after every absorb -while over budget - - - -safe - -ack collector + transaction state -choose safe restart point -stay behind open or unfinished work - - - -buf->safe - - -open transactions - - - -subt->merge - - -(top + subxids) -authoritative on COMMIT - - - -spill - - - -{spill}/xid-{xid:010}-{first_lsn:016X}.bin -["WS" magic | u16 ver=6] -[tag u8 | u32 LE inner_len | body]* -0 Heap · 1 Chunk · 2 ToastDelete · -3 Raw · 4 descriptor dict -append-only, fsync on finish - - - -abort->spill - - -unlink - - - -abort->safe - - - - - -idle->safe - - - - - -evict->spill - - -SpillWriter::write - - - -spill->merge - - -SpillReader::next -then unlink - - - -dispatch - -bounded committed batches -one shared walk keeps rows, TOAST, -and table changes in WAL order - - - -merge->dispatch - - - - - -detoast - -resolve large values -current transaction first, -mirror history second - - - -dispatch->detoast - - -rows - - - -consumer - -shared consumers -live pipeline: ClickHouse or metrics only -backup replay: same ordered walk - - - -dispatch->consumer - - -table changes - - - -store - - -TOAST mirror -save changes before publishing commit -interleave table wipes and rewrites - - - -dispatch->store - - -TOAST changes - - - -dispatch->safe - - -drained - - - -detoast->consumer - - -decoded rows - - - -consumer->safe - - -downstream completed - - - -manifest - - - -manifest.toml -saved restart state - - - -safe->manifest - - -status update - - - -legend - - - -node fill — role - - - -source PG / XLOG_XACT records - - - -eviction policy (sync, after absorb) - - - -XactBuffer / decoder / drain - - - -ShadowCatalog schema_event_tx - - - -downstream consumer (pipeline / gap replay) - - - -on-disk artifact (spill / manifest) - - -edge colour - -━━ - -buffer-internal control / data - -┄┄ - -schema event (lsn-stamped) - -━━ - -walk steps to pipeline / replay consumers - -┄┄ - -spill file IO (write / read / unlink) - -··· - -ack + manifest durability - - -k-way merge — ordering rules - -heads sorted ASC by source_lsn - -tie: catalog event BEFORE tuple (PG writes pg_class first) - -heaps detoast post-merge; chunk maps resolve rows; ToastRows persist separately - -saved restart point never passes open or unfinished transaction - - - - diff --git a/plans/INDEX.md b/plans/INDEX.md index e297e512..6a281a7c 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -58,16 +58,13 @@ invariants which code cannot express ## Architecture diagrams -Live under [architecture/](../architecture/README.md). System-level -SVGs cover overview, internals, shadow communication, bootstrap -timeline, streaming timeline, restart timelines. Component SVGs cover -filter, source, shadow, decoder, xact, TOAST, emitter, bootstrap, ops, -and oracle. Updated on architecturally load-bearing changes +[architecture](../architecture/README.md) owns diagrams shared by these plans: +[streaming topology](../architecture/overview.svg), +[worker pools](../architecture/workers.svg), +[catalog capture and DDL](../architecture/catalog.svg), +[TOAST and type conversion](../architecture/values.svg), +[bootstrap](../architecture/bootstrap.svg), and +[restart and cleanup](../architecture/recovery.svg) -## Regenerating diagrams - -Each `architecture/.dot` carries its own regeneration spec as a -header comment (sources of truth, subsumed plan section, quality bar); -shared style invariants live in [`architecture/palette.md`](../architecture/palette.md). -Workflow in [`architecture/README.md`](../architecture/README.md#regenerating-a-diagram). -Use when regenerating a component diagram after material code change +Update those SVG sources when component connections change; embed them here +instead of maintaining separate diagrams diff --git a/plans/TOAST.md b/plans/TOAST.md index 7de5ecff..d5a008b7 100644 --- a/plans/TOAST.md +++ b/plans/TOAST.md @@ -5,7 +5,7 @@ User-facing mode selection and large-value behavior live in This note covers chunk identity, as-of reconstruction, and replay-safe reclamation. In-xact WAL reassembly is fast path, see [xact.md](xact.md) -![TOAST architecture](../architecture/toast.svg) +![TOAST architecture](../architecture/values.svg) ## Store — TID-keyed mirror diff --git a/plans/bootstrap.md b/plans/bootstrap.md index dcc19718..7f7a9c00 100644 --- a/plans/bootstrap.md +++ b/plans/bootstrap.md @@ -27,8 +27,8 @@ violates catalog-only constraint at [overview.md](overview.md) ## Five-phase greenfield timeline -See [architecture/timeline_bootstrap.svg](../architecture/timeline_bootstrap.svg) -for rendered diagram. Five clusters top→bottom: +See [bootstrap data paths](../architecture/bootstrap.svg) +for data paths and handoff. Bootstrap proceeds through five phases: 1. **Catalog seed** — `walshadow-stream --bootstrap-mode=direct` opens libpq side channel to source PG, runs `seed_catalog_from_source` diff --git a/plans/decoder.md b/plans/decoder.md index 3e32e86f..b7d4d509 100644 --- a/plans/decoder.md +++ b/plans/decoder.md @@ -22,7 +22,7 @@ COLUMN ... DEFAULT k` fast-path missing values reconstructed inline; truly absent bytes flow as `None` columns with `partial = true` for xact buffer to backfill from previous image -![decoder](../architecture/decoder.svg) +![decoder](../architecture/overview.svg) ## Entry point diff --git a/plans/emitter.md b/plans/emitter.md index 22296e90..064b705d 100644 --- a/plans/emitter.md +++ b/plans/emitter.md @@ -38,7 +38,7 @@ commit-record LSN known durable on CH ## Stage walk -![emitter](../architecture/emitter.svg) +![emitter](../architecture/workers.svg) ### Reorder coordinator — `pipeline/reorder.rs` diff --git a/plans/filter.md b/plans/filter.md index fc24c0a6..332160bd 100644 --- a/plans/filter.md +++ b/plans/filter.md @@ -9,7 +9,7 @@ with CRC32C recomputed. Output re-parses through wal-rus `WalParser`; `filtered_lsn == source_lsn` per byte offset, no LSN translation downstream. Manifest sidecar indexes byte positions, not LSN pairs -![filter](../architecture/filter.svg) +![filter](../architecture/overview.svg) ## Classifier diff --git a/plans/ops.md b/plans/ops.md index 09a938b1..a277a886 100644 --- a/plans/ops.md +++ b/plans/ops.md @@ -36,7 +36,7 @@ publishes persisted floor to pruners, then sends standby-status triple to source. Publishing only after durable write makes every GC cut no newer than restart position -![ops](../architecture/ops.svg) +![ops](../architecture/recovery.svg) ## Standby-status triple diff --git a/plans/oracle.md b/plans/oracle.md index af637b7d..a6bf7f10 100644 --- a/plans/oracle.md +++ b/plans/oracle.md @@ -2,7 +2,7 @@ [`src/ops/oracle.rs`](../src/ops/oracle.rs) plus [`pgext/`](../pgext/) -![oracle](../architecture/oracle.svg) +![oracle](../architecture/values.svg) ## Purpose diff --git a/plans/shadow.md b/plans/shadow.md index fd08ecf9..b5dfe0c0 100644 --- a/plans/shadow.md +++ b/plans/shadow.md @@ -102,7 +102,7 @@ duplicate connection state for no measurable win ## Three channels to shadow -See [architecture/shadow_communication.dot](../architecture/shadow_communication.dot) +See [shadow communication](../architecture/overview.svg) for rendered diagram: 1. **libpq catalog queries** — `ShadowCatalog`'s tokio-postgres client. @@ -144,7 +144,7 @@ pub async fn fetch_overlay_descriptors(&mut self, -> Result>; // uncommitted DDL, see below ``` -![shadow](../architecture/shadow.svg) +![shadow](../architecture/catalog.svg) No cache, no invalidation, no event channel: descriptor history lives in the durable log ([desc_log.md](desc_log.md)); capture calls these diff --git a/plans/source.md b/plans/source.md index 51b7dcdd..314699ee 100644 --- a/plans/source.md +++ b/plans/source.md @@ -11,7 +11,7 @@ xact buffering, oracle all hang off [`RecordBytesSink`](../src/source/wal_stream.rs) traits with no second walk of bytes -![source pipeline](../architecture/source.svg) +![source pipeline](../architecture/overview.svg) ## Purpose diff --git a/plans/xact.md b/plans/xact.md index 7f9e81cc..bf79e516 100644 --- a/plans/xact.md +++ b/plans/xact.md @@ -26,7 +26,7 @@ Descriptor access is a wait-free interval lookup against the durable log ([desc_log.md](desc_log.md)); heaps with `ColumnValue::ExternalToast` resolve their owner descriptor at drain the same way -![xact](../architecture/xact.svg) +![xact](../architecture/workers.svg) ## Buffer shape