From 944caf926dd38a2ebe15e8b907430543d0b6b548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 31 Aug 2026 15:00:05 +0000 Subject: [PATCH 1/9] feat(bdk_electrum_streaming)!: Verify proof-of-work against trusted headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client took the server's word for chain data. Headers were spliced into a checkpoint chain unchecked, and a merkle proof was validated against whatever header the server returned for that height, so an anchor only ever meant "this server says so" — a server could invent a height, hand back a matching fake header, and the proof would pass. `HeaderChain` is a `CheckPoint
` anchored to user-provided trusted headers. Syncing starts above the highest one, so the first header's `prev_blockhash` pins the run to a block the user vouched for. From there every header must match a trusted header at its height, link to the block below it, claim the difficulty consensus requires, and hash below that difficulty's target. A reorg is accepted only if it brings more work than the blocks it replaces, and never if it would displace a trusted block. Anchors become `ProvenAnchor { block_id, pos, merkle }`, keeping the proof rather than discarding it, and `Update` becomes `FullScanResponse`. There is no block time on the anchor: the header it was proved against travels with every update. `UpdateJob` now fetches contiguous runs rather than scattered heights, since a header is only verifiable as part of a chain reaching a block we trust, and gained a backfill: a transaction confirmed below where the chain starts grows it downwards to the nearest trusted block. `Cache::headers` is gone — only the verified chain may hold a header. Rebased from #10, which was written before `ChainJob` was replaced by `UpdateJob`; `header_chain.rs` and `anchor.rs` carry over unchanged. BREAKING CHANGE: anchors are `ProvenAnchor` rather than `ConfirmationBlockTime` and `Update` carries `Header` data, so `chain_update` applies to a `LocalChain
`. `State::new` takes a `HeaderChain` in place of a `CheckPoint`, and `Cache::headers` is removed. `bdk_core`/`bdk_chain` now come from git master, `miniscript` is 13, and `rust-version` is 1.85. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 455 +++----------- bdk_electrum_streaming/Cargo.toml | 10 +- bdk_electrum_streaming/README.md | 14 + bdk_electrum_streaming/src/anchor.rs | 33 + bdk_electrum_streaming/src/cache.rs | 63 +- .../src/confirmation_job.rs | 361 +++++------ bdk_electrum_streaming/src/header_chain.rs | 585 ++++++++++++++++++ bdk_electrum_streaming/src/lib.rs | 20 +- bdk_electrum_streaming/src/spk_job.rs | 10 +- bdk_electrum_streaming/src/state.rs | 127 ++-- bdk_electrum_streaming/tests/env.rs | 349 +++++------ bdk_electrum_streaming/tests/state.rs | 406 ++++++------ 12 files changed, 1304 insertions(+), 1129 deletions(-) create mode 100644 bdk_electrum_streaming/src/anchor.rs create mode 100644 bdk_electrum_streaming/src/header_chain.rs diff --git a/Cargo.lock b/Cargo.lock index 7d86ac4..f92e793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -8,17 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.12" @@ -60,21 +49,20 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.1" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] -name = "base64ct" -version = "1.8.3" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bdk_chain" -version = "0.23.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c290eff038799a8ac0c5a82b6160a9ca456baa299a6f22b262c771342d2846c0" +version = "0.23.2" +source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_core", "bitcoin", @@ -83,9 +71,8 @@ dependencies = [ [[package]] name = "bdk_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3028782f6bf14a6df987244333d34e6b272b5a40a53e4879ec2dfd82275a3a" +version = "0.6.2" +source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bitcoin", "hashbrown", @@ -115,10 +102,10 @@ dependencies = [ [[package]] name = "bdk_testenv" version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543dc273dab3b9ec329772bcb15741a948cc3510deae2a30af3a116f03505fee" +source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_chain", + "bitcoin", "electrsd", ] @@ -135,6 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0ce8bd5baaa0d303a19915a6d93afed161f528654e42da2a7a97d05c59499a" dependencies = [ "base58ck", + "base64 0.21.7", "bech32", "bitcoin-io", "bitcoin-units", @@ -192,42 +180,19 @@ dependencies = [ "serde", ] -[[package]] -name = "bitcoincore-rpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" -dependencies = [ - "bitcoincore-rpc-json", - "jsonrpc", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "bitcoincore-rpc-json" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" -dependencies = [ - "bitcoin", - "serde", - "serde_json", -] - [[package]] name = "bitcoind" -version = "0.36.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce6620b7c942dbe28cc49c21d95e792feb9ffd95a093205e7875ccfa69c2925" +checksum = "deae3e9d99e37df8e82023e9f253cabe4b26f8a24a3e4e415f97a546f5280b66" dependencies = [ "anyhow", "bitcoin_hashes", - "bitcoincore-rpc", + "bitreq", + "corepc-client", "flate2", "log", - "minreq", + "serde_json", "tar", "tempfile", "which", @@ -247,12 +212,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "bitreq" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "b65c2e1ab98050c93cc9ada070d5070e014329c65196d03f6f8f1f75c2666f37" dependencies = [ - "generic-array", + "rustls", + "rustls-webpki", + "serde", + "serde_json", + "webpki-roots", ] [[package]] @@ -294,8 +263,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -306,28 +273,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "cipher" -version = "0.4.4" +name = "corepc-client" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "168e5075162cb8c253e8eea1575391c2ef8adde050bf9f3ef29d1cb109364388" dependencies = [ - "crypto-common", - "inout", + "bitcoin", + "corepc-types", + "jsonrpc", + "log", + "serde", + "serde_json", ] [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "corepc-types" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "1583872320eb2ac629c36753023fd072f1ca1b3b74b20cc62bab055b54278789" dependencies = [ - "libc", + "bitcoin", + "serde", + "serde_json", ] [[package]] @@ -345,60 +312,27 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "either" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" - [[package]] name = "electrsd" -version = "0.28.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c3c57645202a05a47206ed81cc179cf32bf4c3ca5a9e60c06c49d0222d5844" +checksum = "50d22a2f7bac981d425536cbeca21bc65b9d11a375c6e9ac853ccf61a1eb6119" dependencies = [ "bitcoin_hashes", "bitcoind", + "bitreq", + "corepc-client", "electrum-client", "log", - "minreq", "nix", - "which", "zip", ] [[package]] name = "electrum-client" -version = "0.20.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7b1f8783238bb18e6e137875b0a66f3dffe6c7ea84066e05d033cf180b150f" +checksum = "a5059f13888a90486e7268bbce59b175f5f76b1c55e5b9c568ceaa42d2b8507c" dependencies = [ "bitcoin", "log", @@ -556,16 +490,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -577,17 +501,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -622,57 +535,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - [[package]] name = "jsonrpc" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +checksum = "f106cca655869522988b976127096f0aa36e0f1a8ff1f1f425a5c85b61e59391" dependencies = [ - "base64", - "minreq", + "base64 0.22.1", + "bitreq", "serde", "serde_json", ] @@ -689,12 +565,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -724,12 +594,13 @@ dependencies = [ [[package]] name = "miniscript" -version = "12.3.7" +version = "13.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8343cc1ef1408bd9bdbf69f7aef47017dfab7e6349ec26fddf62e0e9fb5a4cf" +checksum = "cd35e2c377504e50159561884b03610711db6c2fec6c89f2b98d94016684726b" dependencies = [ "bech32", "bitcoin", + "hex-conservative 1.2.0", ] [[package]] @@ -742,19 +613,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "minreq" -version = "2.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05015102dad0f7d61691ca347e9d9d9006685a64aefb3d79eecf62665de2153d" -dependencies = [ - "rustls", - "rustls-webpki", - "serde", - "serde_json", - "webpki-roots", -] - [[package]] name = "mio" version = "1.2.2" @@ -789,41 +647,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -842,12 +671,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -875,12 +698,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "rand" version = "0.8.7" @@ -908,7 +725,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.17", + "getrandom", ] [[package]] @@ -919,25 +736,12 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.17", + "getrandom", "libc", "untrusted", "windows-sys 0.52.0", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -947,39 +751,41 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] -name = "rustls-webpki" -version = "0.101.7" +name = "rustls-pki-types" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ - "ring", - "untrusted", + "zeroize", ] [[package]] -name = "sct" -version = "0.7.1" +name = "rustls-webpki" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1047,28 +853,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -1158,9 +942,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -1173,25 +956,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "time" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - [[package]] name = "tokio" version = "1.53.1" @@ -1289,12 +1053,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -1327,20 +1085,20 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "webpki-roots" -version = "0.25.4" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "which" -version = "4.4.2" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", + "libc", ] [[package]] @@ -1358,15 +1116,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1447,7 +1196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1470,24 +1219,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zip" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes", "byteorder", "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd", ] [[package]] @@ -1495,32 +1243,3 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/bdk_electrum_streaming/Cargo.toml b/bdk_electrum_streaming/Cargo.toml index 45021b4..20759a2 100644 --- a/bdk_electrum_streaming/Cargo.toml +++ b/bdk_electrum_streaming/Cargo.toml @@ -4,7 +4,7 @@ version = "0.7.0" description = "An async/blocking Electrum client for BDK, built as an explicit streaming state machine." license = "MIT OR Apache-2.0" edition = "2021" -rust-version = "1.70" +rust-version = "1.85" repository = "https://github.com/evanlinjin/experiments" documentation = "https://docs.rs/bdk_electrum_streaming" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" futures = "0.3" futures-timer = "3" anyhow = "1" -bdk_core = { version = "0.6", features = ["serde"] } -miniscript = { version = "12.0.0" } +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master", features = ["serde"] } +bdk_chain = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +miniscript = { version = "13.0.0" } electrum_streaming_client = { version = "0.4" } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" tracing = "0.1" [dev-dependencies] -bdk_testenv = "0.13.0" -bdk_chain = "0.23.0" +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } tokio = { version = "1", features = ["time", "net", "rt", "macros"]} tokio-util = { version = "0.7.15", features = ["compat"] } tracing-subscriber = "0.3" diff --git a/bdk_electrum_streaming/README.md b/bdk_electrum_streaming/README.md index 296d96a..b7b4453 100644 --- a/bdk_electrum_streaming/README.md +++ b/bdk_electrum_streaming/README.md @@ -6,3 +6,17 @@ state machine on top of [`electrum_streaming_client`](https://docs.rs/electrum_s Tracks a set of descriptors, subscribes to their script hashes, and streams `FullScanResponse` updates as history, transactions and anchors resolve, instead of blocking until an entire scan completes. Handles reorgs by refetching whichever anchors they affect. + +The server is not trusted for chain data. You hand [`HeaderChain`] a set of trusted block headers; +everything above the highest one is downloaded and checked for linkage, proof-of-work, and the +difficulty consensus requires before it becomes part of the chain. A reorg is only accepted if it +brings more work than the blocks it replaces, and never if it would drop a trusted block. +Transactions are merkle-proved against those headers, and the proof travels with the anchor +([`ProvenAnchor`]). + +The highest trusted block must sit on a difficulty-adjustment boundary (`height % 2016 == 0`) on +networks where difficulty moves, so every retarget above it can be recomputed rather than taken on +faith. + +A transaction confirmed below the sync start triggers a backfill: headers are fetched from just +above the highest trusted block below it, up to where the chain already begins. diff --git a/bdk_electrum_streaming/src/anchor.rs b/bdk_electrum_streaming/src/anchor.rs new file mode 100644 index 0000000..adafa18 --- /dev/null +++ b/bdk_electrum_streaming/src/anchor.rs @@ -0,0 +1,33 @@ +use bdk_chain::Anchor; +use bdk_core::BlockId; +use electrum_streaming_client::DoubleSHA; + +/// Anchors a transaction to a block, recording the merkle proof that put it there. +/// +/// A [`ProvenAnchor`] only exists if [`merkle`](Self::merkle) was checked against the `merkle_root` +/// of the block's header, and that header is part of the verified +/// [`HeaderChain`](crate::HeaderChain). +/// +/// There is no block time here: the header the proof was checked against travels with every +/// [`Update`](crate::Update) as part of `chain_update`, so times can be read from there. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct ProvenAnchor { + /// The block the transaction is proven to be in. + pub block_id: BlockId, + /// Position of the transaction within the block's merkle tree. + pub pos: usize, + /// Merkle branch connecting the transaction to the block's merkle root. + pub merkle: Vec, +} + +impl Anchor for ProvenAnchor { + fn anchor_block(&self) -> BlockId { + self.block_id + } + + fn confirmation_height_upper_bound(&self) -> u32 { + self.block_id.height + } +} diff --git a/bdk_electrum_streaming/src/cache.rs b/bdk_electrum_streaming/src/cache.rs index a8844fb..eeb2cda 100644 --- a/bdk_electrum_streaming/src/cache.rs +++ b/bdk_electrum_streaming/src/cache.rs @@ -3,11 +3,10 @@ use std::{ sync::Arc, }; -use bdk_core::{ - bitcoin::{self, block::Header, BlockHash, Transaction, Txid}, - ConfirmationBlockTime, -}; -use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; +use bdk_core::bitcoin::{self, BlockHash, Transaction, Txid}; +use electrum_streaming_client::{response, ElectrumScriptHash, ElectrumScriptStatus}; + +use crate::ProvenAnchor; /// Everything learned from the server, kept so a reconnect need not ask again. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -21,9 +20,6 @@ pub struct Cache { /// would only give the two something to disagree about. Seed it from wallet data instead. #[serde(skip)] pub tx_cache: TxCache, - - /// This can be removed once we can place `Header`s in `CheckPoint`s. - pub headers: HashMap, } /// The transaction data a job consults before asking the server for anything. @@ -49,38 +45,7 @@ pub struct TxCache { /// Written as a sequence: a `(Txid, BlockHash)` key is not a string, so a map would be /// unserializable in JSON and every other format that requires string keys. #[serde(with = "persist::anchors_as_seq")] - pub anchors: HashMap<(Txid, BlockHash), ConfirmationBlockTime>, -} - -impl Cache { - pub fn resolve_headers_query( - &mut self, - req: request::Headers, - resp: response::HeadersResp, - ) -> impl Iterator { - self.headers - .extend(resp.headers.iter().map(|&h| (h.block_hash(), h))); - (req.start_height..).zip(resp.headers) - } - - pub fn resolve_history_query( - &mut self, - req: request::GetHistory, - resp: Vec, - ) -> Option { - let status_opt = ElectrumScriptStatus::from_history(&resp); - if let Some(status) = status_opt { - self.tx_cache - .spk_txids - .entry(req.script_hash) - .or_default() - .extend(resp.iter().map(|tx| tx.txid())); - self.subscriptions.insert_spk(req.script_hash, status, resp); - } else { - self.subscriptions.remove_spk(req.script_hash); - } - status_opt - } + pub anchors: HashMap<(Txid, BlockHash), ProvenAnchor>, } /// The last history the server reported for each script hash. @@ -280,7 +245,7 @@ mod persist { use super::*; use serde::{Deserialize, Deserializer, Serializer}; - type Anchors = HashMap<(Txid, BlockHash), ConfirmationBlockTime>; + type Anchors = HashMap<(Txid, BlockHash), ProvenAnchor>; pub fn serialize( anchors: &Anchors, @@ -297,7 +262,7 @@ mod persist { deserializer: D, ) -> Result { Ok( - Vec::<(Txid, BlockHash, ConfirmationBlockTime)>::deserialize(deserializer)? + Vec::<(Txid, BlockHash, ProvenAnchor)>::deserialize(deserializer)? .into_iter() .map(|(txid, block_hash, anchor)| ((txid, block_hash), anchor)) .collect(), @@ -392,9 +357,17 @@ mod test { fn tx_cache_round_trips_through_json() { let anchor = (txid(1), bitcoin::BlockHash::from_byte_array([2; 32])); let mut before = TxCache::default(); - before - .anchors - .insert(anchor, ConfirmationBlockTime::default()); + before.anchors.insert( + anchor, + ProvenAnchor { + block_id: bdk_core::BlockId { + height: 2, + hash: bitcoin::BlockHash::from_byte_array([2; 32]), + }, + pos: 0, + merkle: Vec::new(), + }, + ); let json = serde_json::to_string(&before).expect("must serialize"); let after: TxCache = serde_json::from_str(&json).expect("must deserialize"); diff --git a/bdk_electrum_streaming/src/confirmation_job.rs b/bdk_electrum_streaming/src/confirmation_job.rs index 0c5714f..ed33a55 100644 --- a/bdk_electrum_streaming/src/confirmation_job.rs +++ b/bdk_electrum_streaming/src/confirmation_job.rs @@ -1,20 +1,24 @@ use std::collections::{BTreeMap, BTreeSet}; use bdk_core::{ - bitcoin::{block::Header, BlockHash, Txid}, + bitcoin::{block::Header, Txid}, BlockId, CheckPoint, }; use electrum_streaming_client::{request, ElectrumScriptStatus}; -use crate::{AnchorUpdate, Cache, ReqQueuer}; +use crate::{AnchorUpdate, Cache, HeaderChain, ReqQueuer}; /// How far along [`ConfirmationJob`] is. #[derive(Debug, Default, Clone)] pub enum ConfirmationStage { #[default] Init, - FetchBlocks { - to_fetch: BTreeSet, + /// Waiting on contiguous runs of headers, each `start -> end` inclusive. + /// + /// Runs, not scattered heights: [`HeaderChain`] verifies each header against the one below + /// it, so a header is only worth having as part of an unbroken run reaching the chain. + FetchHeaders { + runs: BTreeMap, }, FetchAnchors { to_fetch: BTreeSet<(u32, Txid)>, @@ -26,31 +30,14 @@ pub enum ConfirmationStage { Done, /// Nothing left to do until the target tip or the statuses move. /// - /// The update was taken, or the job was abandoned on inconsistent headers. Distinct from - /// [`Done`], which still owes one — a single stage for both would hand the same update over - /// twice, and hand one over for an abandoned job. + /// The update was taken, or the job was abandoned on a chain the server has left. Distinct + /// from [`Done`], which still owes one — a single stage for both would hand the same update + /// over twice, and hand one over for an abandoned job. /// /// [`Done`]: Self::Done Idle, } -impl ConfirmationStage { - pub fn fetch_anchors( - cache: &Cache, - spk_statuses: impl IntoIterator, - ) -> Self { - let to_fetch = cache - .subscriptions - .spk_histories(spk_statuses) - .filter_map(|tx| { - let conf_height = tx.confirmation_height()?.to_consensus_u32(); - Some((conf_height, tx.txid())) - }) - .collect(); - Self::FetchAnchors { to_fetch } - } -} - /// What one [`ConfirmationJob::poll`] achieved. /// /// The two `Update` variants are the parts of an [`Update`] this job owns; the rest come from @@ -58,12 +45,13 @@ impl ConfirmationStage { /// /// [`Update`]: crate::Update /// [`SpkJob`]: crate::SpkJob +#[derive(Debug)] pub enum ConfirmationProgress { - /// The local chain moved. - CheckPointUpdate { - cp: CheckPoint, - /// If there are any evictions, we need to check which spks need reanchoring - evicted: Vec, + /// The verified chain moved. + ChainUpdate { + cp: CheckPoint
, + /// Whether the run displaced blocks we already had. + reorged: bool, }, /// Every anchor the job set out to prove, resolved against one chain. AnchorUpdate(AnchorUpdate), @@ -75,7 +63,7 @@ pub enum ConfirmationProgress { Done, } -/// The single job that moves the local chain and anchors what the scripts found. +/// The single job that moves the verified chain and anchors what the scripts found. /// /// Runs once every [`SpkJob`] has its history — the heights those histories name are all it /// reads, so a script still downloading its own transactions has already told it every block it @@ -93,16 +81,28 @@ pub struct ConfirmationJob { target_header: Header, target_statuses: BTreeSet, - /// Always contains the target header; the notification carries it, so it is never fetched. + /// Headers as the server gave them, before [`HeaderChain`] has verified any of them. + /// + /// Kept apart from the chain deliberately: nothing here has been checked, and a header only + /// becomes part of the chain once its whole run passes. Always contains the target header; + /// the notification carries it, so it is never fetched. fetched_headers: BTreeMap, stage: ConfirmationStage, } impl ConfirmationJob { - /// An assumption of the max reorg depth. - const MAX_REORG_DEPTH: u32 = 21; + /// How far below the tip to re-download so a reorg is noticed. + /// + /// A reorg deeper than this is not walked back to; the run will not link to what we hold and + /// the connection errors out rather than quietly keeping a chain the server has left. The + /// verified chain is what stops a deep fork being adopted wrongly — it still has to out-work + /// what it replaces — but noticing one at all stops here. + /// + /// ponytail: fixed 21-block window, downloading every header and verifying work from genesis + /// removes the need for a window at all. + const REORG_WINDOW: u32 = 21; - /// Number of blocks before difficulty adjustment. + /// Most headers a server will hand over in one `blockchain.block.headers`. const MAX_BATCH_HEADERS_REQUEST: u32 = 2016; pub fn new(target_height: u32, target_header: Header) -> Self { @@ -196,156 +196,97 @@ impl ConfirmationJob { self.fetched_headers.extend(blocks); } - /// Polls the job as far as it will go. + /// Take one step towards a verified chain that covers everything worth anchoring. pub fn poll( &mut self, queuer: &mut ReqQueuer, cache: &Cache, - cp: &CheckPoint, + chain: &mut HeaderChain, ) -> anyhow::Result { match core::mem::take(&mut self.stage) { ConfirmationStage::Init => { - let to_fetch = self.missing_heights(cache, cp); - - // NOTE: This logic is not perfect and we may duplicate requests due to spk history - // changes between calls to `ConfirmationJob::poll`. Let's not fix it here as we will - // change this crate to download all headers and verify PoW later so there will be - // no need for this logic. - let mut start_height_opt = Option::::None; - let mut iter = to_fetch - .iter() - .copied() - .filter(|h| !self.fetched_headers.contains_key(h)) - .peekable(); - while let Some(h) = iter.next() { - if start_height_opt.is_none() { - start_height_opt = Some(h); - } - let start_height = start_height_opt.expect("must exist"); - if iter.peek().is_some_and(|&next_h| { - next_h <= h.saturating_add(1) - && next_h.saturating_sub(start_height) < Self::MAX_BATCH_HEADERS_REQUEST - }) { - continue; - } - queuer.enqueue(request::Headers { - start_height, - count: (h + 1).saturating_sub(start_height) as usize, - }); - start_height_opt = None; + let runs = self.required_runs(cache, chain); + for (&start, &end) in &runs { + self.queue_gaps(queuer, start, end); } - - self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + self.stage = ConfirmationStage::FetchHeaders { runs }; Ok(ConfirmationProgress::Continue) } - ConfirmationStage::FetchBlocks { to_fetch } => { - if !to_fetch - .iter() - .all(|h| self.fetched_headers.contains_key(h)) - { - self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + ConfirmationStage::FetchHeaders { runs } => { + let complete = runs.iter().all(|(&start, &end)| { + (start..=end).all(|h| self.fetched_headers.contains_key(&h)) + }); + if !complete { + self.stage = ConfirmationStage::FetchHeaders { runs }; return Ok(ConfirmationProgress::Blocked); } - // Headers that disagree mean a reorg landed between fetches; wait to be told. - let mut iter = self - .fetched_headers - .iter() - .rev() - .take((Self::MAX_REORG_DEPTH + 1) as usize) - .peekable(); - while let Some((&height, header)) = iter.next() { - if let Some(&(&prev_height, prev_header)) = iter.peek() { - if prev_height + 1 == height - && prev_header.block_hash() != header.prev_blockhash - { - tracing::info!( - height, - prev_blockhash = header.prev_blockhash.to_string(), - actual_prev_blockhash = prev_header.block_hash().to_string(), - "Fetched headers are inconsistent. Reorg? Abandoning." - ); - self.reset_headers(); - self.stage = ConfirmationStage::Idle; - return Ok(ConfirmationProgress::Blocked); - } + // A run reaching the target must put the announced block at the announced + // height. Anything else is a chain we were never told about — the server has + // moved on, and moving is what makes it announce again, so abandon rather than + // retry. + if let Some(header) = self.fetched_headers.get(&self.target_height) { + if header.block_hash() != self.target_header.block_hash() { + tracing::info!( + height = self.target_height, + announced = self.target_header.block_hash().to_string(), + received = header.block_hash().to_string(), + "Headers describe a chain other than the one announced. Abandoning.", + ); + self.reset_headers(); + self.stage = ConfirmationStage::Idle; + return Ok(ConfirmationProgress::Blocked); } } - // Everything we hold is spliced in from the lowest header up. The target - // header is always one of them, so there is always a run to splice. - let start = self - .fetched_headers - .keys() - .next() - .copied() - .unwrap_or(self.target_height); - let mut extension = BTreeMap::::new(); - let mut base_opt = Option::::None; - for cp in cp.iter() { - if cp.height() < start { - base_opt = Some(cp); - break; - } - extension.insert(cp.height(), cp.hash()); - } - let new_blocks = self - .fetched_headers - .iter() - .map(|(&height, header)| (height, header.block_hash())); - extension.extend(new_blocks); - if extension.get(&0).is_some_and(|&genesis_hash| { - genesis_hash != cp.get(0).expect("genesis must exist").hash() - }) { - return Err(anyhow::anyhow!("server attempted to replace genesis")); + let was = chain.tip().map(|cp| (cp.height(), cp.hash())); + // Ascending, so a backfill run lands before the run that extends the tip — which + // is the order `HeaderChain::apply` needs, since backfill has to reach the base + // the other run may then move. + for (&start, &end) in &runs { + let headers = (start..=end) + .map(|h| self.fetched_headers[&h]) + .collect::>(); + chain.apply(start, headers)?; } - let extension = extension - .into_iter() - .map(|(height, hash)| BlockId { height, hash }); - let cp_update = match base_opt { - Some(base) => base.extend(extension).expect("must not error"), - None => CheckPoint::from_block_ids(extension).expect("must not error"), - }; - - let mut evicted_heights = Vec::::new(); - for cp in cp.iter() { - if cp_update - .get(cp.height()) - .is_some_and(|cp_update| cp_update == cp) - { - break; + let cp = match chain.tip() { + Some(cp) => cp.clone(), + None => { + self.stage = ConfirmationStage::Done; + return Ok(ConfirmationProgress::Done); } - evicted_heights.push(cp.height()); - } + }; + let reorged = + was.is_some_and(|(height, hash)| chain.block_hash(height) != Some(hash)); - self.stage = - ConfirmationStage::fetch_anchors(cache, self.target_statuses.iter().copied()); - Ok(ConfirmationProgress::CheckPointUpdate { - cp: cp_update, - evicted: evicted_heights, - }) + self.stage = self.anchor_stage(cache); + Ok(ConfirmationProgress::ChainUpdate { cp, reorged }) } ConfirmationStage::FetchAnchors { to_fetch } => { let mut resolved = AnchorUpdate::new(); let mut all_resolved = true; + let mut needs_backfill = false; for &(height, txid) in &to_fetch { - let header = match self.fetched_headers.get(&height) { + let header = match chain.header(height) { Some(header) => header, - // Not expected to fire: a changed history moves the status set, which - // sends the job back to `Init` to plan this height. Release goes back and - // fetches rather than assume which block this height holds. + // Below where the chain starts: it has to grow downwards before this + // height can be checked at all, which `Init` plans a run for. + None if height < chain.base_height() => { + needs_backfill = true; + continue; + } + // Above the verified tip. There is nothing to plan: a run only reaches + // the tip the server has announced, and this height is beyond it. The + // announcement that carries it is what re-polls this — going back to + // `Init` here would replan the same unreachable run forever. None => { - debug_assert!( - false, - "history named height {height}, which the header pass did not cover" - ); - self.stage = ConfirmationStage::Init; - return Ok(ConfirmationProgress::Continue); + all_resolved = false; + continue; } }; match cache.tx_cache.anchors.get(&(txid, header.block_hash())) { - Some(&anchor) => { - resolved.insert((anchor, txid)); + Some(anchor) => { + resolved.insert((anchor.clone(), txid)); } None => { all_resolved = false; @@ -353,6 +294,10 @@ impl ConfirmationJob { } } } + if needs_backfill { + self.stage = ConfirmationStage::Init; + return Ok(ConfirmationProgress::Continue); + } if !all_resolved { // The whole set is kept, not just what is left: each pass resolves all of it // afresh against the chain as it stands right then, so a reorg landing @@ -376,60 +321,74 @@ impl ConfirmationJob { } } - /// The heights we still need from the server. + /// The heights carrying a transaction we have to anchor. + fn anchor_heights(&self, cache: &Cache) -> BTreeSet<(u32, Txid)> { + cache + .subscriptions + .spk_histories(self.target_statuses.iter().copied()) + .filter_map(|tx| Some((tx.confirmation_height()?.to_consensus_u32(), tx.txid()))) + .collect() + } + + fn anchor_stage(&self, cache: &Cache) -> ConfirmationStage { + ConfirmationStage::FetchAnchors { + to_fetch: self.anchor_heights(cache), + } + } + + /// The contiguous runs of headers the chain needs before every anchor can be checked. /// - /// Heights whose header is already reachable from `cp` and `cache` are absorbed into - /// `fetched_headers` on the way through, so what comes back is only the gap. - fn missing_heights(&mut self, cache: &Cache, cp: &CheckPoint) -> BTreeSet { - let mut to_fetch = BTreeSet::::new(); + /// At most two: one up to the announced tip, and one backfilling history below where the + /// chain currently starts. Both are runs rather than the individual heights that want them, + /// because a header is only verifiable as part of a chain reaching a block we trust. + fn required_runs(&self, cache: &Cache, chain: &HeaderChain) -> BTreeMap { + let mut runs = BTreeMap::new(); + + let base = chain.base_height(); + let start = match chain.tip_height() { + // Re-download a window below the tip, so a reorg within it is seen at all. Never + // below the base: the run has to stay contiguous with what is already verified. + Some(tip) => base.max(tip.saturating_sub(Self::REORG_WINDOW)), + None => base, + }; + if start <= self.target_height { + runs.insert(start, self.target_height); + } - // Heights the chain itself has to be checked at. Settled first, because a height in - // here is one whose block may be about to be replaced — absorbing it from `cp` below - // would answer the question with the very block under suspicion. - if self.target_tip() != cp.block_id() { - let only_extends_tip = self - .target_height - .checked_sub(1) - .map(|height| { - let hash = self.target_header.prev_blockhash; - BlockId { height, hash } - }) - .is_some_and(|prev| cp.block_id() == prev); - if only_extends_tip { - to_fetch.extend(cp.height() + 1..=self.target_height); - } else { - // Assumes no reorg is deeper than `MAX_REORG_DEPTH`. - let old_tip = cp.height(); - let new_tip = self.target_height; - to_fetch.extend(old_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=old_tip); - to_fetch.extend(new_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=new_tip); + // A transaction confirmed below where the chain starts cannot be checked against it, so + // the chain has to grow downwards to a block we already trust. + if let Some(&(lowest, _)) = self.anchor_heights(cache).iter().next() { + if lowest < base { + let from = chain.trusted_at_or_below(lowest).unwrap_or(0) + 1; + if from < base { + runs.insert(from, base - 1); + } } } - // Heights that carry a transaction to anchor. One the chain already places, and whose - // header we have, needs no request. - let anchor_heights = self - .target_statuses - .iter() - .filter_map(|&spk_status| { - let heights = cache - .subscriptions - .spk_history(spk_status)? - .iter() - .filter_map(|tx| Some(tx.confirmation_height()?.to_consensus_u32())); - Some(heights) - }) - .flatten() - .collect::>(); - for height in anchor_heights { - if !to_fetch.insert(height) { + runs + } + + /// Queue whatever part of `start..=end` we do not already hold, in server-sized batches. + fn queue_gaps(&self, queuer: &mut ReqQueuer, start: u32, end: u32) { + let mut height = start; + while height <= end { + if self.fetched_headers.contains_key(&height) { + height += 1; continue; } - if let Some(&header) = cp.get(height).and_then(|cp| cache.headers.get(&cp.hash())) { - self.fetched_headers.insert(height, header); + let mut count = 0; + while height + count <= end + && count < Self::MAX_BATCH_HEADERS_REQUEST + && !self.fetched_headers.contains_key(&(height + count)) + { + count += 1; } + queuer.enqueue(request::Headers { + start_height: height, + count: count as usize, + }); + height += count; } - - to_fetch } } diff --git a/bdk_electrum_streaming/src/header_chain.rs b/bdk_electrum_streaming/src/header_chain.rs new file mode 100644 index 0000000..c249902 --- /dev/null +++ b/bdk_electrum_streaming/src/header_chain.rs @@ -0,0 +1,585 @@ +use std::collections::BTreeMap; + +use anyhow::{ensure, Context}; +use bdk_core::{ + bitcoin::{ + block::Header, constants::genesis_block, params::Params, BlockHash, CompactTarget, Work, + }, + CheckPoint, +}; + +/// A chain of block headers, verified against a set of user-provided trusted headers. +/// +/// Every applied header must: +/// 1. match the trusted header at its height (if the user provided one), +/// 2. link to the block below it via `prev_blockhash`, +/// 3. claim the difficulty that consensus requires at its height, and +/// 4. hash below the target of that difficulty. +/// +/// Syncing starts one block above the highest trusted block, so (2) pins the very first header we +/// download to a block the user vouched for. Everything from there up is verified, contiguous, and +/// kept in memory. A reorg on top of that is only accepted if it brings more work than the blocks +/// it replaces. +/// +/// Genesis is trusted implicitly (it is derived from `params`, not downloaded) and every trusted +/// block is included in the [`CheckPoint`] handed out by [`tip`](Self::tip), so it can always be +/// connected to a `LocalChain`. Between the trusted blocks and the sync start there are gaps. +/// +/// # Difficulty +/// +/// (3) is what makes (4) mean anything: without it a server could claim a trivial difficulty and +/// mine a fake chain cheaply. Recomputing a retarget needs the header that opened the previous +/// difficulty period, so on networks where difficulty actually moves, the highest trusted block is +/// required to sit on a difficulty-adjustment boundary (`height % 2016 == 0`). That way every +/// retarget above it is recomputed from a header we already have, with no gap taken on faith. +/// +/// Trusted blocks *below* the sync start are exempt: a backfilled run is pinned by a trusted block +/// at the bottom and the verified chain at the top, so its difficulty needs no checking. +#[derive(Debug, Clone)] +pub struct HeaderChain { + params: Params, + /// Headers the user vouches for, plus genesis. Never downloaded, never replaced. + trusted: BTreeMap, + /// Lowest height of the contiguous verified segment. + base: u32, + cp: Option>, +} + +impl HeaderChain { + /// Construct a [`HeaderChain`] that trusts `trusted` (height to header). + /// + /// The highest entry decides where syncing starts, and must sit on a difficulty-adjustment + /// boundary on networks where difficulty moves. Lower entries are what allow history *below* + /// the sync start to be verified later, when a transaction turns out to be confirmed down + /// there; they can be at any height. + /// + /// Genesis is added automatically; an entry at height `0` must agree with `params`. + pub fn new( + params: impl Into, + trusted: impl IntoIterator, + ) -> anyhow::Result { + let params = params.into(); + let genesis = genesis_block(¶ms).header; + let mut trusted = trusted.into_iter().collect::>(); + if let Some(header) = trusted.insert(0, genesis) { + ensure!( + header.block_hash() == genesis.block_hash(), + "trusted block at height 0 is {}, but {} has genesis {}", + header.block_hash(), + params.network, + genesis.block_hash(), + ); + } + let anchor = *trusted + .keys() + .next_back() + .expect("genesis was just inserted"); + let interval = params.difficulty_adjustment_interval() as u32; + if retargets(¶ms) { + ensure!( + anchor % interval == 0, + "the highest trusted block must sit on a difficulty-adjustment boundary (a \ + multiple of {interval}) so that every retarget above it can be recomputed; \ + height {anchor} is not one", + ); + } + Ok(Self { + params, + trusted, + base: anchor + 1, + cp: None, + }) + } + + fn interval(&self) -> u32 { + self.params.difficulty_adjustment_interval() as u32 + } + + /// The lowest height we need to download a header for. + /// + /// This starts one block above the highest trusted block and moves down as history is + /// backfilled. + pub fn base_height(&self) -> u32 { + self.base + } + + /// The highest trusted height at or below `height`. + pub fn trusted_at_or_below(&self, height: u32) -> Option { + self.trusted.range(..=height).next_back().map(|(&h, _)| h) + } + + /// The verified tip, if we have one. + /// + /// Every trusted block is in it, so it can be applied to a `LocalChain`. + pub fn tip(&self) -> Option<&CheckPoint
> { + self.cp.as_ref() + } + + /// Height of the verified tip, if we have one. + pub fn tip_height(&self) -> Option { + self.cp.as_ref().map(CheckPoint::height) + } + + /// The trusted or verified header at `height`, if we have it. + pub fn header(&self, height: u32) -> Option
{ + if let Some(&header) = self.trusted.get(&height) { + return Some(header); + } + self.cp.as_ref()?.get(height).map(|cp| cp.data()) + } + + /// The trusted or verified blockhash at `height`, if we have it. + pub fn block_hash(&self, height: u32) -> Option { + self.header(height).map(|h| h.block_hash()) + } + + /// Apply a contiguous, ascending run of `headers` beginning at `start`. + /// + /// The run may extend the tip, replace it (reorg), or sit below the current + /// [`base_height`](Self::base_height) to backfill history — in which case it must reach up to + /// the existing base. + /// + /// The chain is left untouched if anything fails to verify. + pub fn apply(&mut self, start: u32, headers: Vec
) -> anyhow::Result<()> { + if headers.is_empty() { + return Ok(()); + } + ensure!(start > 0, "genesis is never applied"); + self.verify(start, &headers)?; + + let end = start + headers.len() as u32 - 1; + let old_tip_height = self.tip_height(); + let run = (start..).zip(headers); + // Trusted blocks below the run keep the checkpoint connectable to a `LocalChain`. + let below = self + .trusted + .range(..start) + .map(|(&height, &header)| (height, header)) + .collect::>(); + + let cp = match self.cp.clone() { + // Backfill: rebuild as the trusted blocks, the run, then whatever sat above the run. + Some(cp) if start < self.base => { + ensure!( + end + 1 >= self.base, + "backfilled headers stop at {end}, below the chain base {}", + self.base + ); + let above = cp + .iter() + .take_while(|cp| cp.height() > end) + .collect::>(); + build( + below + .into_iter() + .chain(run) + .chain(above.into_iter().rev().map(|cp| (cp.height(), cp.data()))), + )? + } + // Extend the tip, evicting any block the run disagrees with. + Some(old) => { + ensure!( + start <= old.height() + 1, + "headers starting at {start} would leave a gap above tip {}", + old.height() + ); + let cp = run.fold(old.clone(), |cp, (height, header)| { + cp.insert(height, header) + }); + // Blocks below `start` are untouched, so comparing the chains from there is the + // same as comparing their totals. + let evicts_old_tip = cp + .get(old.height()) + .is_none_or(|cp| cp.hash() != old.hash()); + if evicts_old_tip { + ensure!( + work_from(&cp, start) > work_from(&old, start), + "the run reorgs our chain from height {start} without more work than the \ + blocks it replaces", + ); + } + cp + } + // Nothing yet: the trusted blocks plus the run become the chain. + None => build(below.into_iter().chain(run))?, + }; + + // An eviction must never take out a block we already trust. + if let Some(old_tip_height) = old_tip_height { + for (&height, header) in self.trusted.range(..=old_tip_height) { + let hash = header.block_hash(); + ensure!( + cp.get(height).is_some_and(|cp| cp.hash() == hash), + "the applied headers displace the trusted block at height {height} ({hash})" + ); + } + } + + self.base = self.base.min(start); + self.cp = Some(cp); + Ok(()) + } + + fn verify(&self, start: u32, headers: &[Header]) -> anyhow::Result<()> { + // Look up a header by height, preferring the run being verified over what we hold. + let at = |height: u32| -> Option
{ + height + .checked_sub(start) + .and_then(|i| headers.get(i as usize).copied()) + .or_else(|| self.header(height)) + }; + + for (i, header) in headers.iter().enumerate() { + let height = start + i as u32; + let hash = header.block_hash(); + + if let Some(trusted) = self.trusted.get(&height) { + ensure!( + hash == trusted.block_hash(), + "block {hash} at height {height} conflicts with trusted block {}", + trusted.block_hash(), + ); + } + if let Some(prev) = height.checked_sub(1).and_then(at) { + let prev_hash = prev.block_hash(); + ensure!( + header.prev_blockhash == prev_hash, + "block {hash} at height {height} does not link to {prev_hash} below it" + ); + } + if let Some(bits) = self.required_bits(height, at) { + ensure!( + header.bits == bits, + "block {hash} at height {height} claims difficulty {:#x}, consensus requires {:#x}", + header.bits.to_consensus(), + bits.to_consensus(), + ); + } + let target = header.target(); + ensure!( + target <= self.params.max_attainable_target, + "block {hash} at height {height} claims a target above the proof-of-work limit" + ); + header + .validate_pow(target) + .with_context(|| format!("block {hash} at height {height}"))?; + } + Ok(()) + } + + /// The difficulty consensus requires at `height`. + /// + /// Difficulty is fixed for a whole retarget period and recomputed at each boundary. Returns + /// `None` when the rule cannot be enforced: on networks that allow min-difficulty blocks, or + /// when we are missing a header the calculation needs — which, above the trusted anchor, + /// cannot happen, since the anchor sits on a boundary. + fn required_bits( + &self, + height: u32, + at: impl Fn(u32) -> Option
, + ) -> Option { + if self.params.allow_min_difficulty_blocks { + return None; + } + let prev = at(height.checked_sub(1)?)?; + let interval = self.interval(); + if height % interval != 0 { + return Some(prev.bits); + } + let boundary = at(height.checked_sub(interval)?)?; + Some(CompactTarget::from_header_difficulty_adjustment( + boundary, + prev, + &self.params, + )) + } +} + +/// Whether difficulty actually moves on this network. +fn retargets(params: &Params) -> bool { + !params.allow_min_difficulty_blocks && !params.no_pow_retargeting +} + +/// Total work of `cp` from `from` up to its tip. +fn work_from(cp: &CheckPoint
, from: u32) -> Work { + cp.range(from..) + .fold(Work::from_be_bytes([0; 32]), |sum, cp| { + sum + cp.data().work() + }) +} + +fn build(blocks: impl IntoIterator) -> anyhow::Result> { + CheckPoint::from_blocks(blocks).map_err(|_| anyhow::anyhow!("headers do not form a chain")) +} + +#[cfg(test)] +mod test { + use super::*; + use bdk_core::bitcoin::{block::Version, hashes::Hash, CompactTarget, TxMerkleNode}; + + /// Regtest, but with the difficulty rules switched on so they actually get exercised. + /// Retargeting stays off, so trusted blocks may sit anywhere. + fn params() -> Params { + let mut params = Params::REGTEST; + params.allow_min_difficulty_blocks = false; + params + } + + /// Regtest with real retargeting over a 10-block period, so retargets are cheap to mine. + fn retarget_params() -> Params { + let mut params = params(); + params.no_pow_retargeting = false; + params.pow_target_timespan = 10 * params.pow_target_spacing; + params + } + + /// Append `n` mined headers to `chain` (indexed by height), retargeting per `params`. + /// + /// `tag` distinguishes otherwise-identical forks. `bits` overrides the difficulty, which only + /// makes sense on a network that allows min-difficulty blocks. + fn extend( + params: &Params, + chain: &mut Vec
, + n: usize, + tag: u8, + bits: Option, + ) { + let interval = params.difficulty_adjustment_interval() as u32; + for _ in 0..n { + let height = chain.len() as u32; + let prev = chain[height as usize - 1]; + let bits = bits.unwrap_or(if height % interval == 0 { + CompactTarget::from_header_difficulty_adjustment( + chain[(height - interval) as usize], + prev, + params, + ) + } else { + prev.bits + }); + let mut merkle_root = [0u8; 32]; + merkle_root[0] = tag; + merkle_root[1] = height as u8; + let mut header = Header { + version: Version::ONE, + prev_blockhash: prev.block_hash(), + merkle_root: TxMerkleNode::from_byte_array(merkle_root), + time: prev.time + 600, + bits, + nonce: 0, + }; + // Grind for real proof-of-work. + while header.validate_pow(header.target()).is_err() { + header.nonce += 1; + } + chain.push(header); + } + } + + /// Genesis plus `n` mined headers. `chain[h]` is the header at height `h`. + fn mine(params: &Params, n: usize) -> Vec
{ + let mut chain = vec![genesis_block(params).header]; + extend(params, &mut chain, n, 0, None); + chain + } + + /// A fork of `chain` that branches above `from`, `n` blocks long. + fn fork( + params: &Params, + chain: &[Header], + from: u32, + n: usize, + bits: Option, + ) -> Vec
{ + let mut forked = chain[..=from as usize].to_vec(); + extend(params, &mut forked, n, 1, bits); + forked[from as usize + 1..].to_vec() + } + + fn chain(headers: &[Header], trusted_heights: &[u32]) -> HeaderChain { + HeaderChain::new( + params(), + trusted_heights.iter().map(|&h| (h, headers[h as usize])), + ) + .unwrap() + } + + #[test] + fn accepts_a_valid_chain() { + let headers = mine(¶ms(), 10); + let mut c = chain(&headers, &[5]); + assert_eq!(c.base_height(), 6, "sync starts above the trusted block"); + c.apply(6, headers[6..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(10)); + assert_eq!(c.header(7), Some(headers[7])); + assert_eq!( + c.tip().unwrap().iter().last().unwrap().height(), + 0, + "genesis is always the base" + ); + assert_eq!( + c.tip().unwrap().get(5).map(|cp| cp.hash()), + Some(headers[5].block_hash()), + "trusted blocks are in the checkpoint" + ); + } + + #[test] + fn rejects_a_run_that_does_not_link_to_the_trusted_block() { + let headers = mine(¶ms(), 10); + let forked = fork(¶ms(), &headers, 4, 6, None); + let mut c = chain(&headers, &[5]); + let err = c.apply(6, forked[1..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("does not link to"), "{err}"); + } + + #[test] + fn rejects_bad_pow() { + let mut headers = mine(¶ms(), 3); + let mut c = chain(&headers, &[]); + while headers[2].validate_pow(headers[2].target()).is_ok() { + headers[2].nonce = headers[2].nonce.wrapping_add(1); + } + let err = c.apply(1, headers[1..3].to_vec()).unwrap_err().to_string(); + assert!(err.contains("height 2"), "{err}"); + assert!(c.tip().is_none()); + } + + #[test] + fn rejects_broken_link() { + let mut headers = mine(¶ms(), 3); + let mut c = chain(&headers, &[]); + headers[3].prev_blockhash = BlockHash::all_zeros(); + assert!(c.apply(1, headers[1..].to_vec()).is_err()); + } + + #[test] + fn rejects_difficulty_change_within_a_period() { + let mut headers = mine(¶ms(), 2); + let mut c = chain(&headers, &[]); + headers[2].bits = CompactTarget::from_consensus(0x207ffffe); + let err = c.apply(1, headers[1..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("consensus requires"), "{err}"); + } + + #[test] + fn rejects_conflict_with_trusted_blockhash() { + let headers = mine(¶ms(), 4); + let forked = fork(¶ms(), &headers, 1, 3, None); + let mut c = chain(&headers, &[3]); + let err = c.apply(2, forked).unwrap_err().to_string(); + assert!(err.contains("conflicts with trusted"), "{err}"); + } + + #[test] + fn rejects_reorg_that_displaces_a_trusted_block() { + // Plain regtest: min-difficulty blocks are allowed, so a fork may be harder than the + // chain it replaces without tripping the difficulty rule. + let params = Params::REGTEST; + let headers = mine(¶ms, 10); + let mut c = HeaderChain::new(params.clone(), [(3, headers[3]), (8, headers[8])]).unwrap(); + c.apply(9, headers[9..].to_vec()).unwrap(); + c.apply(4, headers[4..9].to_vec()).unwrap(); + assert_eq!(c.base_height(), 4); + + // Two blocks at 4x the difficulty out-work the seven they replace, and say nothing about + // height 8 — but they would drop it. + let harder = CompactTarget::from_consensus(0x201f_ffff); + let forked = fork(¶ms, &headers, 3, 2, Some(harder)); + let err = c.apply(4, forked).unwrap_err().to_string(); + assert!(err.contains("displace the trusted block"), "{err}"); + assert_eq!(c.tip_height(), Some(10), "chain is left untouched"); + assert_eq!(c.header(5), Some(headers[5])); + } + + #[test] + fn accepts_a_reorg_with_more_work() { + let headers = mine(¶ms(), 6); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + let forked = fork(¶ms(), &headers, 4, 3, None); + c.apply(5, forked.clone()).unwrap(); + assert_eq!(c.tip_height(), Some(7)); + assert_eq!(c.header(5), Some(forked[0])); + } + + #[test] + fn rejects_a_reorg_with_less_work() { + let headers = mine(¶ms(), 8); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + // Every block here carries the same work, so a shorter fork is a weaker chain. + let forked = fork(¶ms(), &headers, 4, 2, None); + let err = c.apply(5, forked).unwrap_err().to_string(); + assert!(err.contains("without more work"), "{err}"); + assert_eq!(c.tip_height(), Some(8), "chain is left untouched"); + assert_eq!(c.header(6), Some(headers[6])); + } + + #[test] + fn re_applying_the_same_headers_is_not_a_reorg() { + let headers = mine(¶ms(), 8); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + // The reorg window re-downloads blocks we already have; equal work must still be fine. + c.apply(5, headers[5..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(8)); + } + + #[test] + fn backfills_below_the_base() { + let headers = mine(¶ms(), 12); + let mut c = chain(&headers, &[3, 8]); + c.apply(9, headers[9..].to_vec()).unwrap(); + assert_eq!(c.base_height(), 9); + assert_eq!(c.header(5), None); + assert_eq!(c.trusted_at_or_below(5), Some(3)); + + c.apply(4, headers[4..9].to_vec()).unwrap(); + assert_eq!(c.base_height(), 4); + assert_eq!(c.tip_height(), Some(12)); + assert_eq!(c.header(5), Some(headers[5])); + assert_eq!(c.tip().unwrap().iter().last().unwrap().height(), 0); + } + + #[test] + fn rejects_backfill_that_does_not_reach_the_base() { + let headers = mine(¶ms(), 12); + let mut c = chain(&headers, &[3, 8]); + c.apply(9, headers[9..].to_vec()).unwrap(); + assert!(c.apply(4, headers[4..7].to_vec()).is_err()); + } + + #[test] + fn rejects_a_trusted_anchor_off_the_retarget_boundary() { + let params = retarget_params(); + let headers = mine(¶ms, 12); + let err = HeaderChain::new(params, [(11, headers[11])]) + .unwrap_err() + .to_string(); + assert!(err.contains("difficulty-adjustment boundary"), "{err}"); + } + + #[test] + fn verifies_every_retarget_above_a_boundary_anchor() { + let params = retarget_params(); + let headers = mine(¶ms, 25); + assert_ne!( + headers[20].bits, headers[19].bits, + "difficulty must actually move for this test to mean anything" + ); + + let mut c = HeaderChain::new(params.clone(), [(10, headers[10])]).unwrap(); + assert_eq!(c.base_height(), 11); + c.apply(11, headers[11..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(25)); + + // The retarget at 20 is recomputed from the trusted header at 10 — no gap on faith. + let mut faked = headers.clone(); + faked[20].bits = headers[19].bits; + let mut c = HeaderChain::new(params, [(10, headers[10])]).unwrap(); + let err = c.apply(11, faked[11..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("height 20"), "{err}"); + assert!(err.contains("consensus requires"), "{err}"); + } +} diff --git a/bdk_electrum_streaming/src/lib.rs b/bdk_electrum_streaming/src/lib.rs index fbaccc8..2918dca 100644 --- a/bdk_electrum_streaming/src/lib.rs +++ b/bdk_electrum_streaming/src/lib.rs @@ -2,10 +2,13 @@ use std::collections::BTreeSet; -use bdk_core::{bitcoin::Txid, spk_client::FullScanResponse}; +use bdk_core::{ + bitcoin::{block::Header, Txid}, + spk_client::FullScanResponse, +}; +/// Re-export. pub use electrum_streaming_client; -use bdk_core::ConfirmationBlockTime; mod cache; pub use cache::*; mod state; @@ -26,9 +29,18 @@ mod blocking_client; pub use blocking_client::*; mod confirmation_job; pub use confirmation_job::*; +mod header_chain; +pub use header_chain::*; +mod anchor; +pub use anchor::*; -pub type Update = FullScanResponse; -pub type AnchorUpdate = BTreeSet<(ConfirmationBlockTime, Txid)>; +/// What a sync produces. +/// +/// Anchors are [`ProvenAnchor`]s — merkle-proved against a header in the verified +/// [`HeaderChain`] — and the chain update carries full [`Header`]s, so a block's time is read +/// from there rather than copied onto every anchor. +pub type Update = FullScanResponse; +pub type AnchorUpdate = BTreeSet<(ProvenAnchor, Txid)>; pub type BlockingClientAction = ClientAction>; pub type AsyncClientAction = ClientAction; diff --git a/bdk_electrum_streaming/src/spk_job.rs b/bdk_electrum_streaming/src/spk_job.rs index 0e62b36..b9ff34f 100644 --- a/bdk_electrum_streaming/src/spk_job.rs +++ b/bdk_electrum_streaming/src/spk_job.rs @@ -5,11 +5,11 @@ use std::{ use bdk_core::{ bitcoin::{OutPoint, Txid}, - ConfirmationBlockTime, TxUpdate, + TxUpdate, }; use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; -use crate::{req::ReqQueuer, Cache}; +use crate::{req::ReqQueuer, Cache, ProvenAnchor}; /// Where a [`SpkJob`] has got to. /// @@ -62,7 +62,7 @@ pub enum SpkProgress { Blocked, /// Everything asked for has arrived. Carries what the job gathered, leaving it empty, so a /// job polled again after finishing contributes nothing a second time. - Done(TxUpdate), + Done(TxUpdate), } /// The job to perform once we receive a script status notification. @@ -80,7 +80,9 @@ pub struct SpkJob { pub spk_hash: ElectrumScriptHash, stage: SpkStage, - tx_update: TxUpdate, + + /// Staged tx update. + tx_update: TxUpdate, } impl SpkJob { diff --git a/bdk_electrum_streaming/src/state.rs b/bdk_electrum_streaming/src/state.rs index 648e925..d99e302 100644 --- a/bdk_electrum_streaming/src/state.rs +++ b/bdk_electrum_streaming/src/state.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use anyhow::Context; -use bdk_core::{CheckPoint, ConfirmationBlockTime}; +use bdk_core::{bitcoin::block::Header, BlockId}; use electrum_streaming_client::{ notification::Notification, request, AsyncPendingRequest, BlockingPendingRequest, ElectrumScriptHash, ElectrumScriptStatus, MaybeBatch, PendingRequest, @@ -15,7 +15,7 @@ use crate::{ confirmation_job::{ConfirmationJob, ConfirmationProgress}, req::{JobRequest, PoppedRequest, ReqCoord, ReqQueue}, spk_job::{SpkJob, SpkProgress}, - DerivedSpkTracker, Update, + DerivedSpkTracker, HeaderChain, ProvenAnchor, Update, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -41,7 +41,7 @@ pub type BlockingState = State; pub struct State { spk_tracker: DerivedSpkTracker, coord: ReqCoord, - cp: CheckPoint, + chain: HeaderChain, cache: Cache, spk_jobs: BTreeMap, @@ -61,12 +61,12 @@ impl State { coord: ReqCoord, cache: Cache, spk_tracker: DerivedSpkTracker, - cp: CheckPoint, + chain: HeaderChain, ) -> Self { Self { spk_tracker, coord, - cp, + chain, cache, spk_jobs: BTreeMap::new(), confirmation_job: None, @@ -83,6 +83,11 @@ impl State { &self.cache.subscriptions } + /// Get a reference to the verified header chain. + pub fn chain(&self) -> &HeaderChain { + &self.chain + } + /// Insert a descriptor and queue outgoing requests (if needed). pub fn insert_descriptor( &mut self, @@ -163,7 +168,7 @@ impl State { self.spk_jobs.clear(); let update = core::mem::take(&mut self.staged); tracing::info!( - tip_height = self.cp.height(), + tip_height = self.chain.tip_height(), anchors = update.tx_update.anchors.len(), txs = update.tx_update.txs.len(), "Confirmation job finished" @@ -243,18 +248,27 @@ impl State { JobRequest::GetHeaders(req) => { let resp = from_raw(&req, raw)?; debug_assert!(job_ids.contains(&JobId::Confirmation)); - let blocks = self - .cache - .resolve_headers_query(req, resp) - .collect::>(); if let Some(job) = &mut self.confirmation_job { - job.resolve_blocks(blocks); + job.resolve_blocks((req.start_height..).zip(resp.headers)); } self.poll_confirmation_job(req_queue) } JobRequest::GetHistory(req) => { let resp = from_raw(&req, raw)?; - let resp_status = self.cache.resolve_history_query(req, resp); + let resp_status = ElectrumScriptStatus::from_history(&resp); + if let Some(spk_status) = resp_status { + self.cache + .tx_cache + .spk_txids + .entry(req.script_hash) + .or_default() + .extend(resp.iter().map(|tx| tx.txid())); + self.cache + .subscriptions + .insert_spk(req.script_hash, spk_status, resp); + } else { + self.cache.subscriptions.remove_spk(req.script_hash); + } // A history that does not hash to the status a job awaits can never // satisfy it, and the two differing means the status moved — so a @@ -300,12 +314,12 @@ impl State { return self.poll_confirmation_job(req_queue); } - let cp = match self.cp.get(req.height) { - Some(cp) => cp, - // Not expected to fire: the job places every height before it asks - // a proof for it, and a height leaving the chain bumps the generation - // the check above catches. Getting here is our own bookkeeping - // breaking, not the server misbehaving. + let header = match self.chain.header(req.height) { + Some(header) => header, + // Not expected to fire: the job places every height before it asks a + // proof for it, and a height leaving the verified chain bumps the + // generation the check above catches. Getting here is our own + // bookkeeping breaking, not the server misbehaving. None => { debug_assert!( false, @@ -315,28 +329,7 @@ impl State { tracing::error!( ?req, ?resp, - "Received a merkle proof before we placed the block" - ); - self.confirmation_job = None; - return Ok(()); - } - }; - let header = match self.cache.headers.get(&cp.hash()) { - Some(header) => *header, - // Not expected either, and a reorg is not the reason — that is the - // check above. Every header a job puts in the chain lands in - // `Cache::headers` as it arrives, and nothing prunes them. - None => { - debug_assert!( - false, - "no header for {}, the block we hold at height {}", - cp.hash(), - req.height - ); - tracing::error!( - ?req, - blockhash = cp.hash().to_string(), - "No header for the block we hold at this height", + "Proof for a height the verified chain does not reach", ); self.confirmation_job = None; return Ok(()); @@ -352,9 +345,13 @@ impl State { ); self.cache.tx_cache.anchors.insert( (req.txid, header.block_hash()), - ConfirmationBlockTime { - block_id: cp.block_id(), - confirmation_time: header.time as u64, + ProvenAnchor { + block_id: BlockId { + height: req.height, + hash: header.block_hash(), + }, + pos: resp.pos, + merkle: resp.merkle, }, ); } else { @@ -385,17 +382,7 @@ impl State { } /// React to the server announcing `header` at `height` as its tip. - fn on_new_tip( - &mut self, - req_queue: &mut ReqQueue, - height: u32, - header: bdk_core::bitcoin::block::Header, - ) -> anyhow::Result<()> { - // A same-height reorg is applied without fetching anything, so this announcement is the - // only place the replacement header is ever offered to us. Caching it here saves the - // anchor refetch a round-trip on the very path it exists for. - self.cache.headers.insert(header.block_hash(), header); - + fn on_new_tip(&mut self, req_queue: &mut ReqQueue, height: u32, header: Header) -> anyhow::Result<()> { match &mut self.confirmation_job { Some(job) => { if job.set_tip(height, header) { @@ -452,18 +439,17 @@ impl State { // A notification is all that revives a cancelled job, and below the reorg window the // tip never moves — so this is where an anchor the server has come back to is picked up. if self.confirmation_job.is_none() { - match self.cache.headers.get(&self.cp.hash()) { - Some(&header) => { - self.confirmation_job = Some(ConfirmationJob::new(self.cp.height(), header)); + match self.chain.tip() { + Some(cp) => { + self.confirmation_job = Some(ConfirmationJob::new(cp.height(), cp.data())); } - // Not expected to fire: a tip is only adopted through a notification, which - // caches its header. Ask for the tip rather than leave the anchors waiting on a - // block ten minutes out; a request already in flight absorbs this one. + // Not expected to fire: a tip is only adopted through a notification, and the + // very first one builds the chain's initial run before any spk status can + // arrive. Ask for the tip rather than leave the anchors waiting on a block ten + // minutes out; a request already in flight absorbs this one. None => { tracing::warn!( - tip_height = self.cp.height(), - tip_hash = self.cp.hash().to_string(), - "No header for our tip, so no confirmation job can be built. Resubscribing." + "No verified tip yet, so no confirmation job can be built. Resubscribing." ); self.coord .queuer(req_queue, JobId::Confirmation) @@ -550,7 +536,7 @@ impl State { loop { let progress = { let mut queuer = self.coord.queuer(req_queue, JobId::Confirmation); - match job.poll(&mut queuer, &self.cache, &self.cp) { + match job.poll(&mut queuer, &self.cache, &mut self.chain) { Ok(progress) => progress, Err(err) => { self.confirmation_job = Some(job); @@ -560,17 +546,16 @@ impl State { }; match progress { ConfirmationProgress::Continue => continue, - ConfirmationProgress::CheckPointUpdate { cp, evicted } => { - if !evicted.is_empty() { + ConfirmationProgress::ChainUpdate { cp, reorged } => { + if reorged { tracing::info!( - heights = ?evicted, - "Blocks evicted from the local chain. Refetching anchors." + tip_height = cp.height(), + "Blocks displaced from the verified chain. Refetching anchors." ); - // Responses to requests which are still in flight describe the chain we - // just left behind. + // Responses to requests which are still in flight were made against the + // chain we just left behind. self.coord.bump_chain_generation(); } - self.cp = cp.clone(); self.staged.chain_update = Some(cp); continue; } diff --git a/bdk_electrum_streaming/tests/env.rs b/bdk_electrum_streaming/tests/env.rs index 9b62998..223cac3 100644 --- a/bdk_electrum_streaming/tests/env.rs +++ b/bdk_electrum_streaming/tests/env.rs @@ -1,18 +1,19 @@ use std::{sync::atomic::AtomicBool, time::Duration}; +use std::collections::BTreeMap; + use bdk_chain::{ - keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, CanonicalizationParams, - ChainPosition, IndexedTxGraph, + keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, ChainPosition, IndexedTxGraph, }; -use bdk_core::{ - bitcoin::{key::Secp256k1, params::REGTEST, Address, Amount, BlockHash, Txid}, - ConfirmationBlockTime, +use bdk_core::bitcoin::{ + block::Header, constants::genesis_block, key::Secp256k1, params::REGTEST, Address, Amount, + BlockHash, Network, Txid, }; use bdk_electrum_streaming::{ run_async, run_blocking, AsyncClient, AsyncState, BlockingClient, BlockingState, Cache, - DerivedSpkTracker, ReqCoord, Update, + DerivedSpkTracker, HeaderChain, ProvenAnchor, ReqCoord, Update, }; -use bdk_testenv::{bitcoincore_rpc::RpcApi, utils::DESCRIPTORS, TestEnv}; +use bdk_testenv::{electrsd::electrum_client::ElectrumApi, utils::DESCRIPTORS, TestEnv}; use futures::{channel::mpsc, pin_mut, FutureExt, StreamExt}; use miniscript::Descriptor; use tokio::net::TcpStream; @@ -23,6 +24,8 @@ const EXTERNAL: &str = "external"; const INTERNAL: &str = "internal"; const LOOKAHEAD: u32 = 6; +type Graph = IndexedTxGraph>; + fn init() { let _ = tracing_subscriber::fmt() .with_test_writer() @@ -30,51 +33,76 @@ fn init() { .try_init(); } +fn genesis_header() -> Header { + genesis_block(®TEST).header +} + fn apply_update( - chain: &mut LocalChain, - graph: &mut IndexedTxGraph>, + chain: &mut LocalChain
, + graph: &mut Graph, update: Update<&'static str>, ) -> anyhow::Result<()> { let _ = graph .index .reveal_to_target_multi(&update.last_active_indices); let _ = graph.apply_update(update.tx_update); - if let Some(cp) = update.chain_update { - chain.apply_update(cp)?; - } else { - panic!("NO CHAIN UPDATE!"); - } + let cp = update.chain_update.expect("NO CHAIN UPDATE!"); + chain.apply_update(cp)?; Ok(()) } -#[test] -fn blocking_env() -> anyhow::Result<()> { - init(); - +/// Set up an indexer/graph/chain trio plus the spk tracker fed to the client. +fn setup() -> anyhow::Result<(Graph, LocalChain
, DerivedSpkTracker<&'static str>)> { let secp = Secp256k1::new(); - let env = TestEnv::new()?; - let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; + let (external, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; + let (internal, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - let mut graph = IndexedTxGraph::::new({ + let graph = IndexedTxGraph::new({ let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); indexer.insert_descriptor(EXTERNAL, external.clone())?; indexer.insert_descriptor(INTERNAL, internal.clone())?; indexer }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let (chain, _) = LocalChain::from_genesis(genesis_header()); let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); spk_tracker.insert_descriptor(EXTERNAL, external, 0); spk_tracker.insert_descriptor(INTERNAL, internal, 0); + Ok((graph, chain, spk_tracker)) +} + +fn confirmed_balance(chain: &LocalChain
, graph: &Graph) -> bdk_chain::Balance { + // `LocalChain` only canonicalizes over blockhashes for now, so drop the headers. + let chain = LocalChain::from_blocks( + chain + .iter_checkpoints() + .map(|cp| (cp.height(), cp.hash())) + .collect::>(), + ) + .expect("must build blockhash chain"); + chain + .canonical_view(graph.graph(), chain.tip().block_id(), Default::default()) + .balance( + graph.index.outpoints().clone(), + |(k, _), _| *k == INTERNAL, + 0, + ) +} + +#[test] +fn blocking_env() -> anyhow::Result<()> { + init(); + + let env = TestEnv::new()?; + let electrum_url = env.electrsd.electrum_url.clone(); + let (mut graph, mut chain, spk_tracker) = setup()?; + let mut state = BlockingState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, update_rx) = std::sync::mpsc::channel::>(); @@ -112,17 +140,7 @@ fn blocking_env() -> anyhow::Result<()> { } } - let balance = graph.graph().balance( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - graph.index.outpoints().clone(), - |(k, _), _| *k == INTERNAL, - ); - for cp in chain.iter_checkpoints() { - println!("height={}, hash={}", cp.height(), cp.hash()); - } - println!("BALANCE: {}", balance); + println!("BALANCE: {}", confirmed_balance(&chain, &graph)); // TODO: Figure out a way to stop the thread without having to close the connection. conn.shutdown(std::net::Shutdown::Both)?; @@ -136,30 +154,15 @@ fn blocking_env() -> anyhow::Result<()> { async fn env() -> anyhow::Result<()> { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); - - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + let (mut graph, mut chain, spk_tracker) = setup()?; let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -208,17 +211,7 @@ async fn env() -> anyhow::Result<()> { } } - let balance = graph.graph().balance( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - graph.index.outpoints().clone(), - |(k, _), _| *k == INTERNAL, - ); - for cp in chain.iter_checkpoints() { - println!("height={}, hash={}", cp.height(), cp.hash()); - } - println!("BALANCE: {}", balance); + println!("BALANCE: {}", confirmed_balance(&chain, &graph)); client.stop().await?; run_handle.await??; @@ -226,38 +219,42 @@ async fn env() -> anyhow::Result<()> { Ok(()) } -/// A new block confirming a tracked tx must anchor it on the live connection — no reconnect, -/// regardless of the order in which the server sends the script hash and header notifications. -/// The order-sensitive case (history reporting a height above the local tip) is pinned -/// deterministically in `tests/state.rs`. +/// Trust only the tip, so the wallet's whole history sits below the sync start and can only be +/// anchored by backfilling headers from a trusted block below it. #[tokio::test] -async fn new_block_confirmation_is_anchored_live() -> anyhow::Result<()> { +async fn backfills_history_below_the_sync_start() -> anyhow::Result<()> { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); + let (mut graph, mut chain, spk_tracker) = setup()?; - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); + // Mine everything *before* the client ever connects. + let ((_, spk), _) = graph + .index + .next_unused_spk(EXTERNAL) + .expect("must derive spk"); + env.mine_blocks(101, Some(Address::from_script(&spk, ®TEST)?))?; + env.wait_until_electrum_sees_block(Duration::from_secs(30))?; - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + // The user vouches for this block; take the header from the server but hold it to the hash. + let trusted_height = 101_u32; + let trusted_header = env + .electrum_client() + .block_header(trusted_height as usize)?; + let trusted_hash: BlockHash = env.get_block_hash(trusted_height as u64)?; + assert_eq!(trusted_header.block_hash(), trusted_hash); let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(trusted_height, trusted_header)])?, + ); + assert_eq!( + state.chain().base_height(), + trusted_height + 1, + "sync starts one block above the highest trusted block", ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -277,113 +274,65 @@ async fn new_block_confirmation_is_anchored_live() -> anyhow::Result<()> { anyhow::Ok(()) }); - let update = update_rx.next().await.expect("Must have next update"); - apply_update(&mut chain, &mut graph, update)?; - - // Coinbase maturity, so that `env.send` has funds to spend. - env.mine_blocks(101, None)?; - let premine_height = env.rpc_client().get_block_count()? as u32; - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); pin_mut!(timeout); - while chain.tip().height() < premine_height { - futures::select! { - _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for chain sync.")), - update = update_rx.next() => { - let update = update.expect("Must have next update"); - apply_update(&mut chain, &mut graph, update)?; - }, - } - } - - let ((_, spk), _) = graph - .index - .next_unused_spk(EXTERNAL) - .expect("must derive spk"); - let txid = env.send(&Address::from_script(&spk, ®TEST)?, Amount::ONE_BTC)?; - - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); - pin_mut!(timeout); - loop { - futures::select! { - _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for unconfirmed tx.")), - update = update_rx.next() => { - let update = update.expect("Must have next update"); - let has_tx = update.tx_update.txs.iter().any(|tx| tx.compute_txid() == txid); - apply_update(&mut chain, &mut graph, update)?; - if has_tx { - break; - } - }, - } - } - let confirm_height = env.rpc_client().get_block_count()? as u32 + 1; - env.mine_blocks(1, None)?; - - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); - pin_mut!(timeout); let anchor = loop { futures::select! { - _ = timeout => return Err(anyhow::anyhow!( - "Timed-out waiting for anchor: tx is still unconfirmed at tip height {}", - chain.tip().height(), - )), + _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for the backfill.")), update = update_rx.next() => { let update = update.expect("Must have next update"); + let anchor = update.tx_update.anchors.iter().next().cloned(); apply_update(&mut chain, &mut graph, update)?; - let confirmed_anchor = graph - .graph() - .list_canonical_txs( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - ) - .find(|ctx| ctx.tx_node.txid == txid) - .and_then(|ctx| match ctx.chain_position { - ChainPosition::Confirmed { anchor, .. } => Some(anchor), - ChainPosition::Unconfirmed { .. } => None, - }); - if let Some(anchor) = confirmed_anchor { + if let Some((anchor, _)) = anchor { break anchor; } }, } }; - assert_eq!(anchor.block_id.height, confirm_height); + + assert!( + anchor.block_id.height <= trusted_height, + "the coinbase we found is below the sync start, at {}", + anchor.block_id.height, + ); + assert!( + !anchor.merkle.is_empty() || anchor.pos == 0, + "the anchor carries its merkle proof", + ); + assert_eq!( + chain.get(anchor.block_id.height).map(|cp| cp.hash()), + Some(anchor.block_id.hash), + "the backfilled header made it into the chain update", + ); + assert!(confirmed_balance(&chain, &graph).confirmed.to_sat() > 0); client.stop().await?; run_handle.await??; - Ok(()) } -type Graph = IndexedTxGraph>; - -/// The anchor `txid` is canonically confirmed at, if it is confirmed at all. -fn canonical_anchor( - chain: &LocalChain, - graph: &Graph, - txid: Txid, -) -> Option { - graph - .graph() - .list_canonical_txs( - chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - ) - .find(|ctx| ctx.tx_node.txid == txid) - .and_then(|ctx| match ctx.chain_position { - ChainPosition::Confirmed { anchor, .. } => Some(anchor), - ChainPosition::Unconfirmed { .. } => None, - }) +/// The anchor a canonical, confirmed `txid` has — `None` if it is unconfirmed or unknown. +fn canonical_anchor(chain: &LocalChain
, graph: &Graph, txid: Txid) -> Option { + // `LocalChain` only canonicalizes over blockhashes for now, so drop the headers. + let chain = LocalChain::from_blocks( + chain + .iter_checkpoints() + .map(|cp| (cp.height(), cp.hash())) + .collect::>(), + ) + .expect("must build blockhash chain"); + let view = chain.canonical_view(graph.graph(), chain.tip().block_id(), Default::default()); + match view.tx(txid)?.pos { + ChainPosition::Confirmed { anchor, .. } => Some(anchor), + ChainPosition::Unconfirmed { .. } => None, + } } /// A live client against a fresh `electrsd`, with the machinery the reorg tests share. struct LiveWallet { env: TestEnv, - chain: LocalChain, + chain: LocalChain
, graph: Graph, update_rx: mpsc::UnboundedReceiver>, client: AsyncClient<&'static str>, @@ -395,30 +344,15 @@ impl LiveWallet { async fn new() -> anyhow::Result { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); - - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + let (mut graph, mut chain, spk_tracker) = setup()?; let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -451,6 +385,10 @@ impl LiveWallet { }) } + fn block_count(&self) -> anyhow::Result { + Ok(self.env.rpc_client().get_block_count()?.0 as u32) + } + /// Apply updates until `f` holds. /// /// Errors if the client stops — which is what a connection torn down by an expected server @@ -458,7 +396,7 @@ impl LiveWallet { async fn wait_until( &mut self, what: &str, - mut f: impl FnMut(&LocalChain, &Graph) -> bool, + mut f: impl FnMut(&LocalChain
, &Graph) -> bool, ) -> anyhow::Result<()> { let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); pin_mut!(timeout); @@ -483,7 +421,7 @@ impl LiveWallet { /// Returns the txid and the hash of the block confirming it. async fn confirm_tracked_tx(&mut self) -> anyhow::Result<(Txid, BlockHash)> { self.env.mine_blocks(101, None)?; - let premine_height = self.env.rpc_client().get_block_count()? as u32; + let premine_height = self.block_count()?; self.wait_until("the premined chain", |chain, _| { chain.tip().height() >= premine_height }) @@ -522,34 +460,29 @@ impl LiveWallet { /// Issue #12, end to end: a reorg re-mines a confirmed tx into a *different* block at the *same* /// height. The Electrum script status is a hash over txid-height pairs, so it is unchanged and no /// script hash notification is sent. The anchor must still be refetched off the tip alone. +/// +/// The fork also has to out-work the chain it replaces, or the verified chain will not take it — +/// so the replacement run is one block longer, which is what makes a real node switch too. #[tokio::test] async fn reorg_to_same_height_block_refetches_anchor_live() -> anyhow::Result<()> { let mut w = LiveWallet::new().await?; let (txid, first_block) = w.confirm_tracked_tx().await?; - let confirm_height = w.env.rpc_client().get_block_count()? as u32; + let confirm_height = w.block_count()?; // Invalidate the confirming block and re-mine at the same height. The tx is back in the - // mempool, so it goes into the replacement block too. + // mempool, so it goes into the replacement block too. One extra block gives the fork the + // work it needs to be adopted. w.env.reorg(1)?; - let second_block = w.env.rpc_client().get_best_block_hash()?; + w.env.mine_empty_block()?; + let second_block = w + .env + .rpc_client() + .get_block_hash(confirm_height as u64)? + .block_hash()?; assert_ne!( first_block, second_block, "the reorg must actually replace the block" ); - assert_eq!( - w.env.rpc_client().get_block_count()? as u32, - confirm_height, - "the replacement block must be at the same height" - ); - assert!( - w.env - .rpc_client() - .get_block(&second_block)? - .txdata - .iter() - .any(|tx| tx.compute_txid() == txid), - "the replacement block must still contain the tx" - ); w.wait_until("the refetched anchor", |chain, graph| { canonical_anchor(chain, graph, txid).is_some_and(|a| a.block_id.hash == second_block) @@ -569,19 +502,15 @@ async fn reorg_to_same_height_block_refetches_anchor_live() -> anyhow::Result<() async fn reorg_unconfirming_a_tx_keeps_the_connection_alive() -> anyhow::Result<()> { let mut w = LiveWallet::new().await?; let (txid, _) = w.confirm_tracked_tx().await?; - let confirm_height = w.env.rpc_client().get_block_count()? as u32; + let confirm_height = w.block_count()?; // Invalidate the confirming block and replace it with empty ones, so the tx cannot be // re-mined and the server has no proof to give at that height. w.env.invalidate_blocks(1)?; w.env.mine_empty_block()?; w.env.mine_empty_block()?; - let tip_height = w.env.rpc_client().get_block_count()? as u32; + let tip_height = w.block_count()?; assert_eq!(tip_height, confirm_height + 1); - assert!( - w.env.rpc_client().get_raw_mempool()?.contains(&txid), - "the tx must be back in the mempool, so the server really has no proof for it" - ); // The connection has to keep serving: `wait_until` fails if the client stops. w.wait_until("the chain tip after the reorg", |chain, _| { diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index 7bc28a0..fd24ad5 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -9,13 +9,13 @@ use bdk_core::{ transaction, Amount, CompactTarget, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Txid, Witness, }, - BlockId, CheckPoint, ConfirmationBlockTime, + BlockId, }; use bdk_electrum_streaming::{ electrum_streaming_client::{ response, ElectrumScriptHash, ElectrumScriptStatus, RawNotificationOrResponse, RawRequest, }, - BlockingState, Cache, DerivedSpkTracker, ReqCoord, ReqQueue, Update, + BlockingState, Cache, DerivedSpkTracker, HeaderChain, ProvenAnchor, ReqCoord, ReqQueue, Update, }; use miniscript::{Descriptor, DescriptorPublicKey}; use serde_json::json; @@ -152,6 +152,21 @@ fn drain_requests( updates } +/// Drain like [`drain_requests`], but hand back the first error rather than panicking on it. +fn drain_requests_fallible( + state: &mut BlockingState, + queue: &mut ReqQueue, + server: &Server, +) -> anyhow::Result>> { + let mut updates = Vec::new(); + while let Some(req) = queue.pop_front() { + if let Some(update) = state.poll(queue, response(&req, server))? { + updates.push(update); + } + } + Ok(updates) +} + /// Drain like [`drain_requests`], but answer every merkle proof in the queue ahead of everything /// else in each round. /// @@ -222,17 +237,28 @@ fn tx_paying(spk: &ScriptBuf, sats: u64) -> Transaction { } } +/// Grind `nonce` upwards until the header actually clears its own target. +/// +/// Regtest's target is easy but not free: a little under half of all nonces miss it, and the +/// header chain checks proof-of-work for real. +fn mine(mut header: block::Header) -> block::Header { + while header.validate_pow(header.target()).is_err() { + header.nonce = header.nonce.wrapping_add(1); + } + header +} + /// The regtest genesis block and an empty block on top of it. fn base_headers() -> (block::Header, block::Header) { let genesis = constants::genesis_block(Network::Regtest).header; - let header_1 = block::Header { + let header_1 = mine(block::Header { version: block::Version::ONE, prev_blockhash: genesis.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 100, bits: CompactTarget::from_consensus(0x207fffff), nonce: 0, - }; + }); (genesis, header_1) } @@ -243,14 +269,14 @@ fn block_with_root( time: u32, nonce: u32, ) -> block::Header { - block::Header { + mine(block::Header { version: block::Version::ONE, prev_blockhash: prev.block_hash(), merkle_root, time, bits: CompactTarget::from_consensus(0x207fffff), nonce, - } + }) } /// A block whose only transaction is `txid`, so that its merkle root is the txid itself. @@ -258,39 +284,42 @@ fn block_with_tx(prev: &block::Header, txid: Txid, time: u32, nonce: u32) -> blo block_with_root(prev, Txid::to_raw_hash(txid).into(), time, nonce) } -fn new_state( - cache: Cache, - descriptor: Descriptor, - genesis: block::Header, -) -> BlockingState { - new_state_with_cp( - cache, - descriptor, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), - ) +/// A state that trusts nothing but genesis, so everything above it has to be verified. +fn new_state(cache: Cache, descriptor: Descriptor) -> BlockingState { + new_state_trusting(cache, descriptor, []) } -fn new_state_with_cp( +fn new_state_trusting( cache: Cache, descriptor: Descriptor, - cp: CheckPoint, + trusted: impl IntoIterator, ) -> BlockingState { let mut spk_tracker = DerivedSpkTracker::new(0); spk_tracker.insert_descriptor("external", descriptor, 0); - BlockingState::new(ReqCoord::default(), cache, spk_tracker, cp) + let chain = HeaderChain::new(Network::Regtest, trusted).expect("must build header chain"); + BlockingState::new(ReqCoord::default(), cache, spk_tracker, chain) } -/// The anchor a tx confirmed in `header` at `height` must be given. -fn anchor_of(header: &block::Header, height: u32) -> ConfirmationBlockTime { - ConfirmationBlockTime { +/// The anchor a tx confirmed in `header` at `height` must be given, for a server answering with +/// the empty proof that a single-transaction block has. +fn anchor_of(header: &block::Header, height: u32) -> ProvenAnchor { + anchor_proved_by(header, height, Vec::new(), 0) +} + +/// The same, for a server answering with a real merkle branch. +fn anchor_proved_by( + header: &block::Header, + height: u32, + merkle: Vec, + pos: usize, +) -> ProvenAnchor { + ProvenAnchor { block_id: BlockId { height, hash: header.block_hash(), }, - confirmation_time: header.time as u64, + pos, + merkle, } } @@ -310,7 +339,7 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( let mut cache = Cache::default(); cache.tx_cache.txs.insert(txid, Arc::new(tx.clone())); - let mut state = new_state(cache, descriptor, genesis); + let mut state = new_state(cache, descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1], @@ -373,15 +402,11 @@ fn descriptor_inserted_mid_connection_is_subscribed() -> anyhow::Result<()> { let descriptor = Descriptor::::from_str(&format!("wpkh({XPUB}/0/*)"))?; let spk_hash = ElectrumScriptHash::new(descriptor.at_derivation_index(0)?.script_pubkey()); - let genesis = constants::genesis_block(Network::Regtest).header; let mut state = BlockingState::new( ReqCoord::default(), Cache::default(), DerivedSpkTracker::new(0), - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, [])?, ); let mut queue = ReqQueue::new(); @@ -422,10 +447,7 @@ fn last_active_index_is_index_of_active_spk() -> anyhow::Result<()> { ReqCoord::default(), Cache::default(), spk_tracker, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, []).expect("must build header chain"), ); let mut queue = ReqQueue::new(); let server = Server { @@ -480,10 +502,7 @@ fn last_active_index_is_highest_regardless_of_notification_order() -> anyhow::Re ReqCoord::default(), Cache::default(), spk_tracker, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, []).expect("must build header chain"), ); let mut queue = ReqQueue::new(); let mut server = Server { @@ -550,14 +569,10 @@ fn anchor_is_refetched_when_tx_moves_to_another_block_of_same_height() -> anyhow let header_2 = block_with_tx(&header_1, txid, 200, 0); // The block that replaces height 2 contains the tx too, hence the identical script status. let header_2b = block_with_tx(&header_1, txid, 222, 1); - let header_3b = block::Header { - prev_blockhash: header_2b.block_hash(), - time: 300, - ..header_1 - }; + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); assert_ne!(header_2.block_hash(), header_2b.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -626,7 +641,7 @@ fn merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let header_2b = block_with_root(&header_1, proof_2b.expected_merkle_root(txid), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -673,10 +688,13 @@ fn merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let anchors = updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>(); assert!( - anchors.contains(&(anchor_of(&header_2b, 2), txid)), + anchors.contains(&( + anchor_proved_by(&header_2b, 2, proof_2b.merkle.clone(), proof_2b.pos), + txid + )), "the anchor must be refetched rather than written off from a proof of the evicted block" ); Ok(()) @@ -700,7 +718,7 @@ fn anchors_staged_before_a_reorg_are_not_emitted_after_it() -> anyhow::Result<() let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -742,7 +760,7 @@ fn anchors_staged_before_a_reorg_are_not_emitted_after_it() -> anyhow::Result<() let anchors = updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>(); for evicted in [ (anchor_of(&header_2, 2), txid_a), @@ -780,7 +798,7 @@ fn a_tx_unconfirmed_by_a_reorg_does_not_error_the_connection() -> anyhow::Result let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 333, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -834,7 +852,7 @@ fn merkle_error_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -910,7 +928,7 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -970,12 +988,16 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { Ok(()) } -/// Issue #12's literal case: a reorg to a block of the *same* height, with no growth at all. -/// The tip announcement carries the replacement header, so this is the one reorg shape that can -/// be applied without fetching a single block — and every other reorg test here also grows the -/// chain, which takes a different path. +/// A fork of the same height, carrying no more work than the chain it would replace, must be +/// refused — and refusing it must leave the anchor we already have intact. +/// +/// This is what a full node does: an equal-work fork loses to the chain already in hand. The +/// server having moved to it is not evidence, since a server is exactly what the verified chain +/// exists to stop trusting. When the fork does out-work us, the tip announcement that says so is +/// what triggers the switch, and +/// [`anchor_is_refetched_when_tx_moves_to_another_block_of_same_height`] covers that. #[test] -fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { +fn a_fork_without_more_work_is_refused() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); @@ -984,7 +1006,7 @@ fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { let header_2b = block_with_tx(&header_1, txid, 222, 1); assert_ne!(header_2.block_hash(), header_2b.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1004,22 +1026,25 @@ fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { // The tip does not move: same height, different block, unchanged script status. server.headers = vec![genesis, header_1, header_2b]; - state.poll( + let notified = state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", "method": "blockchain.headers.subscribe", "params": [{ "hex": serialize_hex(&header_2b), "height": 2 }], })), - )?; - let updates = drain_requests(&mut state, &mut queue, &server); - + ); + let err = notified + .and_then(|_| drain_requests_fallible(&mut state, &mut queue, &server)) + .expect_err("an equal-work fork must be refused"); assert!( - updates.iter().any(|u| u - .tx_update - .anchors - .contains(&(anchor_of(&header_2b, 2), txid))), - "anchor must be refetched for the block that replaced the evicted one" + format!("{err:#}").contains("without more work"), + "the error must name the reason, got: {err:#}" + ); + assert_eq!( + state.chain().block_hash(2), + Some(header_2.block_hash()), + "the chain must be left on the block it already had" ); Ok(()) } @@ -1039,7 +1064,7 @@ fn anchor_is_refetched_whatever_order_the_server_answers_in() -> anyhow::Result< let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1078,16 +1103,14 @@ fn anchor_is_refetched_whatever_order_the_server_answers_in() -> anyhow::Result< Ok(()) } -/// A client restored from a persisted checkpoint chain starts with blocks in its chain whose -/// headers are not in its cache. Resolving an anchor at such a height needs both the header and -/// the proof, and nothing else will fetch that header — the chain consistency pass has nothing to -/// do, the tip being already correct. +/// A client restored from a trusted block has no verified history below it, so a transaction +/// confirmed down there cannot be anchored until the chain is backfilled to a block it trusts. /// -/// So this is the case where the two halves of the ordering fix are load-bearing: the proof must -/// not be asked for before the header is cached, and the header response must poll the waiting -/// job even though the chain itself has nothing to learn from it. +/// The proof must still not be checked before the header it is verified against arrives, which +/// `drain_requests_proofs_first` forces by answering every merkle request ahead of everything +/// else. #[test] -fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::Result<()> { +fn anchor_below_the_trusted_block_is_backfilled() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); @@ -1095,23 +1118,9 @@ fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::R let header_2 = block_with_tx(&header_1, txid, 200, 0); let header_3 = block_with_root(&header_2, TxMerkleNode::all_zeros(), 300, 0); - // The restored chain knows the blocks, the fresh cache knows none of their headers. The tx - // is one block below the tip, so the header it needs is not the one `headers.subscribe` - // hands back, and the chain consistency pass has nothing to do either, the tip being - // already correct. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: header_2.block_hash(), - }) - .insert(BlockId { - height: 3, - hash: header_3.block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + // Trusting the tip puts the chain base above the transaction, so nothing below it is + // verified and the sync range covers none of it. + let mut state = new_state_trusting(Cache::default(), descriptor, [(3, header_3)]); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -1127,18 +1136,22 @@ fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::R .tx_update .anchors .contains(&(anchor_of(&header_2, 2), txid))), - "the anchor must resolve even when the proof overtakes the header it is verified against" + "the anchor must resolve once history below the trusted block is backfilled" + ); + assert_eq!( + state.chain().base_height(), + 1, + "the chain must have grown down to just above genesis" ); Ok(()) } /// A header batch fetched before a reorg describes the chain we have since left behind, and -/// splicing it in would put a purged block into the checkpoint chain. +/// splicing it in would put a purged block into the verified chain. /// -/// The case that exposes it is a *sparse* chain — a restored one, or one whose missing heights -/// sit below the reorg window `ConfirmationJob` rewrites — reorged deeper than that window, so the -/// consistency pass never learns the low block changed too. Only the height the anchor needs -/// brings it back, and that fetch was in flight when the chain moved. +/// The batch is dropped when the tip that wanted it is abandoned, so the answer never reaches +/// the chain at all — and even if it did, it neither links to the chain we moved to nor +/// out-works it. #[test] fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; @@ -1146,8 +1159,9 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - // Two chains which differ at height 2 as well as near the tip. Both contain the tx at - // height 2, so the anchor stays valid throughout — only the block it belongs to changes. + // Two chains which differ at height 2. Both contain the tx there, so the anchor stays valid + // throughout — only the block it belongs to changes. The second is longer, so it carries + // more work and is allowed to replace the first. let build = |second: block::Header, tip: u32, nonce: u32| { let mut chain = vec![genesis, header_1, second]; for height in 3..=tip { @@ -1161,22 +1175,12 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu } chain }; - let chain_a = build(block_with_tx(&header_1, txid, 200, 0), 30, 0); - let chain_b = build(block_with_tx(&header_1, txid, 222, 1), 31, 1); + let chain_a = build(block_with_tx(&header_1, txid, 200, 0), 8, 0); + let chain_b = build(block_with_tx(&header_1, txid, 222, 1), 9, 1); let (a2, b2) = (chain_a[2], chain_b[2]); assert_ne!(a2.block_hash(), b2.block_hash()); - // A restored chain sparse enough that height 2 is a gap — so the anchor has to fetch that - // header, and `replaces` will not decline it when it comes back. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain_a[30].block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain_a.clone(), @@ -1204,15 +1208,13 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu }; let stale_resp = response(&stale_req, &server); - // The reorg lands. It runs deeper than the reorg window, so the consistency pass rewrites - // only the top of the chain and never learns that height 2 changed too. server.headers = chain_b.clone(); state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", "method": "blockchain.headers.subscribe", - "params": [{ "hex": serialize_hex(&chain_b[31]), "height": 31 }], + "params": [{ "hex": serialize_hex(&chain_b[9]), "height": 9 }], })), )?; let mut updates = drain_requests(&mut state, &mut queue, &server); @@ -1221,21 +1223,15 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu updates.extend(state.poll(&mut queue, stale_resp)?); updates.extend(drain_requests(&mut state, &mut queue, &server)); - let tip = updates - .iter() - .rev() - .find_map(|u| u.chain_update.clone()) - .expect("must get a chain update"); - let at_2 = tip.iter().find(|cp| cp.height() == 2); - assert_ne!( - at_2.as_ref().map(|cp| cp.hash()), - Some(a2.block_hash()), - "a header from the chain we left must not be spliced into the checkpoint chain" - ); assert_eq!( - at_2.map(|cp| cp.hash()), + state.chain().block_hash(2), Some(b2.block_hash()), - "the height must be refetched against the chain we are actually on" + "the height must be verified against the chain we are actually on" + ); + assert_ne!( + state.chain().block_hash(2), + Some(a2.block_hash()), + "a header from the chain we left must not enter the verified chain" ); assert!( updates @@ -1262,7 +1258,7 @@ fn a_replayed_job_does_not_displace_one_the_server_started() -> anyhow::Result<( let header_2b = block_with_tx(&header_1, txid_a, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1341,24 +1337,25 @@ fn a_replayed_job_does_not_displace_one_the_server_started() -> anyhow::Result<( Ok(()) } -/// A persisted checkpoint chain can be stale at a height below the window `ConfirmationJob` rewrites -/// — an offline reorg, say. Then the block *we* have at that height is one the server does not -/// have, and a header request is keyed by height, so no request can ever fetch it. +/// A trusted block the server disagrees with must stop the connection, not spin it. /// -/// Withholding the proof until that header is cached must not turn into an endless request loop. +/// Trusting a block is the user's assertion that it is canonical, so a server offering a +/// different one at that height is not something to reconcile — every header above it descends +/// from a block we do not accept. The old client would ask again forever; this one refuses the +/// run and errors out, and the error names the conflict. #[test] -fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { +fn a_server_disagreeing_with_a_trusted_block_stops_the_connection() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - // The server's height 2, and the stale one our persisted chain still claims. + // The server's height 2, and the one we have been told to trust. let server_2 = block_with_tx(&header_1, txid, 200, 0); - let stale_2 = block_with_tx(&header_1, txid, 999, 7); - assert_ne!(server_2.block_hash(), stale_2.block_hash()); + let trusted_2 = block_with_tx(&header_1, txid, 999, 7); + assert_ne!(server_2.block_hash(), trusted_2.block_hash()); let mut chain = vec![genesis, header_1, server_2]; - for height in 3..=30u32 { + for height in 3..=8u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( &prev, @@ -1368,20 +1365,7 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { )); } - // Tip agrees with the server, so the consistency pass never rewrites height 2. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: stale_2.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut state = new_state_trusting(Cache::default(), descriptor, [(2, trusted_2)]); let mut queue = ReqQueue::new(); let server = Server { headers: chain, @@ -1391,7 +1375,11 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { state.start(&mut queue); let mut served = 0; - while let Some(req) = queue.pop_front() { + let err = loop { + let req = match queue.pop_front() { + Some(req) => req, + None => panic!("the client must not settle while it disagrees with the server"), + }; served += 1; assert!( served < 200, @@ -1399,8 +1387,15 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { req.method, req.params ); - state.poll(&mut queue, response(&req, &server))?; - } + if let Err(err) = state.poll(&mut queue, response(&req, &server)) { + break err; + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("does not link") || msg.contains("trusted"), + "the error must name the conflict, got: {msg}" + ); Ok(()) } @@ -1420,7 +1415,7 @@ fn a_history_that_comes_back_empty_clears_the_subscription() -> anyhow::Result<( let (genesis, header_1) = base_headers(); let header_2 = block_with_tx(&header_1, txid, 200, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1495,7 +1490,7 @@ fn a_script_whose_history_goes_away_is_not_replayed() -> anyhow::Result<()> { let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1578,8 +1573,11 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { let theirs = block_with_root(&header_1, proof_theirs.expected_merkle_root(txid), 222, 1); assert_ne!(ours.merkle_root, theirs.merkle_root); - let mut chain = vec![genesis, header_1, theirs]; - for height in 3..=30u32 { + // The chain the server serves is ours — a block it genuinely disagreed with would be caught + // by header verification long before any proof. What it gets wrong is the *proof*: the + // branch it answers with expands to a root that is not the one in our block. + let mut chain = vec![genesis, header_1, ours]; + for height in 3..=8u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( &prev, @@ -1589,24 +1587,7 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { )); } - // A persisted chain holding our block at height 2, and its header already cached — otherwise - // `GetHeader` catches the disagreement before any proof is asked for. The tip agrees, so no - // chain job runs and nothing rewrites height 2: the disagreement is below the window. - let mut cache = Cache::default(); - cache.headers.insert(ours.block_hash(), ours); - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: ours.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(cache, descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain, @@ -1626,9 +1607,8 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { "a proof for a block we do not have must not anchor anything" ); - // The server comes back to our block at that height. Nothing durable was written against it, - // so the job a notification rebuilds must be able to anchor there. - server.headers[2] = ours; + // The server starts answering with the right proof. Nothing durable was written against the + // bad one, so the job a notification rebuilds must be able to anchor there. server.merkle_proof = (Vec::new(), 0); let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) .expect("history must be non-empty"); @@ -1650,9 +1630,9 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { Ok(()) } -/// A server that will not prove a transaction at a height says our two chains disagree there. It -/// does not say the transaction is absent from our block, and below the reorg window nothing -/// rewrites that height, so the tip never moves and no tip notification is coming. +/// A server erroring a merkle request says nothing about the block itself — the headers agree +/// throughout — so no reorg is coming to explain it, and below the reorg window nothing about +/// the chain ever changes to trigger a refetch either. The tip stays put. /// /// The script notification is the only thing that comes back, and it can only revive a job that /// still exists — [`ConfirmationJob`] is built in `on_new_tip` and nowhere else. Dropping the job @@ -1665,11 +1645,8 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); let ours = block_with_tx(&header_1, txid, 200, 0); - // Their block at that height does not hold the tx, so they will not prove it there. - let theirs = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); - assert_ne!(ours.merkle_root, theirs.merkle_root); - let mut chain = vec![genesis, header_1, theirs]; + let mut chain = vec![genesis, header_1, ours]; for height in 3..=30u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( @@ -1680,26 +1657,9 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( )); } - // As in `a_proof_for_another_block_is_not_a_verdict_on_ours`: our block at height 2 is already - // in the cache and the chain, so `GetHeader` does not catch the disagreement first, and the - // agreeing tip keeps any chain job from rewriting that height. - let mut cache = Cache::default(); - cache.headers.insert(ours.block_hash(), ours); - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: ours.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(cache, descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); - let mut server = Server { + let server = Server { headers: chain, txs: vec![(tx, 2)], merkle_proof: (Vec::new(), 0), @@ -1725,10 +1685,14 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( state.cache().tx_cache.anchors.is_empty(), "an error must not anchor anything" ); + assert_eq!( + state.chain().tip_height(), + Some(30), + "the tip must have synced past the reorg window despite the error" + ); - // The server comes back to our block at that height. The tip is untouched, so this - // notification is the whole of the recovery. - server.headers[2] = ours; + // The tip is untouched and 28 blocks above height 2 — well past the reorg window a tip + // movement would rewrite — so this notification is the whole of the recovery. let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) .expect("history must be non-empty"); state.poll( @@ -1766,7 +1730,7 @@ fn anchor_survives_a_pass_that_happens_after_it_resolved() -> anyhow::Result<()> let (genesis, header_1) = base_headers(); let header_2 = block_with_tx(&header_1, txid, 200, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1, header_2], @@ -1811,7 +1775,7 @@ fn a_tip_that_moves_while_headers_are_in_flight_is_not_lost() -> anyhow::Result< let (chain_a, chain_b) = (build(0), build(1)); assert_ne!(chain_a[3].block_hash(), chain_b[3].block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain_a.clone(), @@ -1884,7 +1848,7 @@ fn headers_for_a_chain_we_were_not_told_about_are_not_adopted() -> anyhow::Resul let b3 = block_with_root(&h2, TxMerkleNode::all_zeros(), 300, 1); assert_ne!(a3.block_hash(), b3.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, h1, h2], @@ -1947,7 +1911,7 @@ fn a_reorg_reanchors_every_script_not_just_the_last_to_notify() -> anyhow::Resul let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -1958,7 +1922,7 @@ fn a_reorg_reanchors_every_script_not_just_the_last_to_notify() -> anyhow::Resul let anchors_of = |updates: &[Update<&'static str>]| { updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>() }; @@ -2026,7 +1990,7 @@ fn a_history_that_cannot_match_the_job_is_not_re_asked() -> anyhow::Result<()> { let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); // The server's chain is one block long, so its history for this script is empty — it will // never answer with the status the notification below carries. @@ -2090,7 +2054,7 @@ fn confirmation_job_runs_ahead_but_the_update_waits_for_the_scripts() -> anyhow: let header_2 = block_with_tx(&header_1, txid, 200, 0); // Deliberately not seeded with the transaction, so the spk job has to ask for it. - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1], @@ -2231,7 +2195,7 @@ fn a_transaction_that_is_not_the_one_asked_for_is_rejected() -> anyhow::Result<( assert_ne!(impostor.compute_txid(), tx.compute_txid()); let (genesis, header_1) = base_headers(); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1], From f2aad11053a034673488f078426faf9da041d87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Mon, 31 Aug 2026 15:28:20 +0000 Subject: [PATCH 2/9] fix(bdk_electrum_streaming): Do not replan a run that can never be fetched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FetchAnchors` read a missing header as history the chain had not backfilled yet and went back to `Init` to plan a run for it. But a run only ever reaches the tip the server has announced, so an anchor *above* that tip has no run to plan: `Init` produced the same runs, `FetchHeaders` completed with nothing to do, and the anchor stage asked again — a loop with no exit inside a single `State::poll`. A history can name a height above the announced tip in the ordinary course of things: electrs notifies the script hash before the header. Distinguish the two misses. Below the chain base is a backfill, which `Init` can plan; above the verified tip is simply pending, and the announcement that carries the tip is what re-polls it. --- bdk_electrum_streaming/tests/state.rs | 68 +++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index fd24ad5..25824c7 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -2032,6 +2032,74 @@ fn a_history_that_cannot_match_the_job_is_not_re_asked() -> anyhow::Result<()> { Ok(()) } +/// A history can name a confirmation height above the tip the server has announced: electrs +/// notifies the script hash before the header. That anchor has to *wait* for the tip to catch +/// up. Treating it as history the chain has not backfilled yet sends the job back to plan a run +/// it can never fetch, and the state machine spins. +#[test] +fn an_anchor_above_the_announced_tip_does_not_spin() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut state = new_state(Cache::default(), descriptor); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.start(&mut queue); + drain_requests(&mut state, &mut queue, &server); + + // The server now has the block, so its history reports the transaction at height 2 — but + // only the script hash is notified, which is the order electrs really uses. + server.headers.push(header_2); + let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history is not empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + drain_requests(&mut state, &mut queue, &server); + + assert_eq!( + state.chain().tip_height(), + Some(1), + "no tip was announced, so the chain must not have moved" + ); + assert!( + state.cache().tx_cache.anchors.is_empty(), + "and nothing may be anchored at a height the chain has not reached" + ); + + // The tip catches up, and only now can the anchor resolve. + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_2), "height": 2 }], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "the anchor must be delivered once the tip catches up" + ); + Ok(()) +} + /// The confirmation job must not wait on transactions it never reads. /// /// It works from the heights a history names, so once every script has its history it already From 7613178a073987a8bd2377f05315b893d7cd1e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 2 Sep 2026 03:40:03 +0000 Subject: [PATCH 3/9] refactor(bdk_electrum_streaming)!: Rebuild the cache from wallet data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything in `Cache` can be rebuilt from what a wallet already knows, so it does not need to round-trip through serde. `Cache::from_wallet_txs` takes `(tx, status, relevant_scripts)` — one entry per transaction, not one per script it pays, since that is how a wallet holds them — and rebuilds all four fields: `subscriptions` and `spk_txids` by computing the same status hash a server would, `txs` from the transaction itself, and `anchors` from the `ProvenAnchor` a confirmed status carries, so nothing has to be redownloaded or reproved on reconnect. With nothing left to persist, `TxCache`'s fields (`spk_txids`, `txs`, `anchors`) move directly onto `Cache`, and the serde-only `persist` module (`HistoryTx`, the anchors-as-seq codec, `Subscriptions`' hand-written `Serialize`/`Deserialize`) is gone. The status hash needs care. Electrum mixes in only `txid:height:`, but orders a history by height *and then block position*, so two transactions confirmed in the same block hash to a status the server would not recognise unless their order matches the block's — `ProvenAnchor::pos`, which a wallet already has from the merkle proof, is what orders them. Mempool entries follow the protocol's `(-height, tx_hash)` rule rather than being left in whatever order the caller supplied. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013Ga32WdTURWnpMDxAckM2K --- bdk_electrum_streaming/src/cache.rs | 484 ++++++++++-------- .../src/confirmation_job.rs | 2 +- bdk_electrum_streaming/src/spk_job.rs | 8 +- bdk_electrum_streaming/src/state.rs | 14 +- bdk_electrum_streaming/tests/state.rs | 10 +- 5 files changed, 284 insertions(+), 234 deletions(-) diff --git a/bdk_electrum_streaming/src/cache.rs b/bdk_electrum_streaming/src/cache.rs index eeb2cda..ee2722f 100644 --- a/bdk_electrum_streaming/src/cache.rs +++ b/bdk_electrum_streaming/src/cache.rs @@ -3,38 +3,21 @@ use std::{ sync::Arc, }; -use bdk_core::bitcoin::{self, BlockHash, Transaction, Txid}; +use bdk_core::bitcoin::{BlockHash, ScriptBuf, Transaction, Txid}; use electrum_streaming_client::{response, ElectrumScriptHash, ElectrumScriptStatus}; use crate::ProvenAnchor; -/// Everything learned from the server, kept so a reconnect need not ask again. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +/// Everything learned from the server, so a job knows what it need not ask for again. +/// +/// Not persisted: every part of it is already in the caller's wallet — rebuild all of it with +/// [`Cache::from_wallet_txs`] — and a stored copy would only give the two something to disagree +/// about. +#[derive(Debug, Clone, Default)] pub struct Cache { /// The server's per-script histories. pub subscriptions: Subscriptions, - /// What we already hold, so a job knows what it need not ask for. - /// - /// Not persisted: every part of it is in the caller's wallet already, and a second copy - /// would only give the two something to disagree about. Seed it from wallet data instead. - #[serde(skip)] - pub tx_cache: TxCache, -} - -/// The transaction data a job consults before asking the server for anything. -/// -/// Separate from the rest of [`Cache`] because a caller can rebuild all of it from their own -/// wallet: the transactions are in their graph, the anchors with them, and which transactions -/// paid a script is what their spk index is for. So none of it is persisted alongside -/// [`Subscriptions`], which nothing can reconstruct. -/// -/// Starting empty is always correct, only expensive: a job asks the server for whatever it -/// cannot find here, so an empty one re-downloads every transaction and reproves every anchor. -/// It is not a mirror of the wallet, though — whatever a job fetches lands here too, so it -/// answers "do we already have this" whoever supplied it. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct TxCache { /// Every txid ever seen for each script hash. /// /// This is monotonically growing so that we can detect evictions. @@ -42,17 +25,109 @@ pub struct TxCache { pub txs: HashMap>, - /// Written as a sequence: a `(Txid, BlockHash)` key is not a string, so a map would be - /// unserializable in JSON and every other format that requires string keys. - #[serde(with = "persist::anchors_as_seq")] pub anchors: HashMap<(Txid, BlockHash), ProvenAnchor>, } +/// A transaction's chain position, as a wallet already tracks it — enough to describe the +/// history a script had, without asking the server. +#[derive(Debug, Clone)] +pub enum TxConfirmationStatus { + /// Confirmed, with the proof a job would otherwise have to ask the server for again. + /// + /// `anchor.pos` plays no part in the status hash itself — only `txid` and `height` are + /// hashed — but a server orders a history by height *and then block position* (per the + /// [protocol]), so two transactions confirmed in the same block hash to a different status + /// if their order is wrong. + /// + /// [protocol]: https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status + Confirmed(ProvenAnchor), + Mempool { + confirmed_inputs: bool, + }, +} + +impl Cache { + /// Rebuild everything from what a wallet already knows: [`subscriptions`](Self::subscriptions), + /// `spk_txids`, `txs` and `anchors`. A reconnect need not download every script's history, + /// refetch its transactions, or reprove its anchors — only whatever actually changed. + /// + /// No `bdk_chain` dependency needed: `txs` is whatever a wallet's tx graph or index already + /// hands over as `(tx, status, relevant_scripts)` — one entry per transaction, not one per + /// script it pays, since that is how a wallet holds them. + pub fn from_wallet_txs( + txs: impl IntoIterator, + ) -> Self + where + Tx: Into>, + Spks: IntoIterator, + { + let mut cache = Cache::default(); + + // Sorted on later, by (height, block position) — carried alongside the response `Tx` + // since the wire type itself has nowhere to put it. + let mut by_spk_hash = HashMap::>::new(); + for (tx, status, relevant_scripts) in txs { + let tx = tx.into(); + let txid = tx.compute_txid(); + let (history_tx, sort_pos) = match status { + TxConfirmationStatus::Confirmed(anchor) => { + let history_tx = response::Tx::Confirmed(response::ConfirmedTx { + txid, + height: bdk_core::bitcoin::absolute::Height::from_consensus( + anchor.block_id.height, + ) + .expect("confirmed tx must have a valid height"), + }); + let pos = anchor.pos; + cache.anchors.insert((txid, anchor.block_id.hash), anchor); + (history_tx, pos) + } + TxConfirmationStatus::Mempool { confirmed_inputs } => ( + response::Tx::Mempool(response::MempoolTx { + txid, + fee: bdk_core::bitcoin::Amount::ZERO, + confirmed_inputs, + }), + 0, + ), + }; + cache.txs.insert(txid, tx); + for spk in relevant_scripts { + by_spk_hash + .entry(ElectrumScriptHash::new(&spk)) + .or_default() + .push((history_tx.clone(), sort_pos)); + } + } + + for (spk_hash, mut entries) in by_spk_hash { + // The order a server reports a history in, and the order its status hash is + // computed over: confirmed ascending by height then block position, then mempool + // ordered by confirmed-inputs before not, each tied by txid. + entries.sort_by_key(|(tx, pos)| match tx { + response::Tx::Confirmed(tx) => (0u8, tx.height.to_consensus_u32(), *pos, tx.txid), + response::Tx::Mempool(tx) if tx.confirmed_inputs => (1, 0, 0, tx.txid), + response::Tx::Mempool(tx) => (2, 0, 0, tx.txid), + }); + let history = entries.into_iter().map(|(tx, _)| tx).collect::>(); + cache + .spk_txids + .entry(spk_hash) + .or_default() + .extend(history.iter().map(response::Tx::txid)); + if let Some(status) = ElectrumScriptStatus::from_history(&history) { + cache.subscriptions.insert_spk(spk_hash, status, history); + } + } + cache + } +} + /// The last history the server reported for each script hash. /// -/// Unlike [`TxCache`], a caller cannot rebuild this from wallet data: a status is a hash Electrum -/// computes over the history it stands for and no wallet stores, and the server reports a history -/// as it stands now, never again mentioning a transaction it has dropped. +/// A caller cannot rebuild this from wallet data alone: a status is a hash Electrum computes +/// over the history it stands for and no wallet stores, so [`Cache::from_wallet_txs`] computes +/// it the same way Electrum does. /// /// Fields are private: a status is a hash of the history it stands for, and letting the two be /// set independently would reintroduce the desync the type exists to prevent. @@ -145,136 +220,10 @@ impl Subscriptions { } } -/// Types and impls that exist only so [`Cache`] can be stored and loaded. -/// -/// Kept apart from the cache itself because [`HistoryTx`] mirrors [`response::Tx`] and the two -/// are easy to mistake for each other at a glance. -mod persist { - use super::*; - - /// A history entry in the shape we can write back out. - /// - /// [`response::Tx`] derives `Deserialize` only, so histories round-trip through this instead. - #[derive(serde::Serialize, serde::Deserialize)] - enum HistoryTx { - Mempool { - txid: Txid, - fee_sats: u64, - confirmed_inputs: bool, - }, - Confirmed { - txid: Txid, - height: u32, - }, - } - - impl From<&response::Tx> for HistoryTx { - fn from(tx: &response::Tx) -> Self { - match tx { - response::Tx::Mempool(tx) => Self::Mempool { - txid: tx.txid, - fee_sats: tx.fee.to_sat(), - confirmed_inputs: tx.confirmed_inputs, - }, - response::Tx::Confirmed(tx) => Self::Confirmed { - txid: tx.txid, - height: tx.height.to_consensus_u32(), - }, - } - } - } - - impl TryFrom for response::Tx { - type Error = bitcoin::absolute::ConversionError; - - fn try_from(tx: HistoryTx) -> Result { - Ok(match tx { - HistoryTx::Mempool { - txid, - fee_sats, - confirmed_inputs, - } => Self::Mempool(response::MempoolTx { - txid, - fee: bitcoin::Amount::from_sat(fee_sats), - confirmed_inputs, - }), - HistoryTx::Confirmed { txid, height } => Self::Confirmed(response::ConfirmedTx { - txid, - height: bitcoin::absolute::Height::from_consensus(height)?, - }), - }) - } - } - - /// Written as `spk_hash -> (status, history)`, which is what rebuilds both maps on the way - /// back in. - impl serde::Serialize for Subscriptions { - fn serialize(&self, serializer: S) -> Result { - serializer.collect_map(self.spk_hash_to_status.iter().map(|(&spk_hash, &status)| { - let history = self - .spk_status_to_history - .get(&status) - .map(|history| history.iter().map(HistoryTx::from).collect::>()) - .unwrap_or_default(); - (spk_hash, (status, history)) - })) - } - } - - impl<'de> serde::Deserialize<'de> for Subscriptions { - fn deserialize>(deserializer: D) -> Result { - use serde::de::Error; - let stored = - HashMap::)>::deserialize( - deserializer, - )?; - let mut spk_histories = Self::default(); - for (spk_hash, (status, history)) in stored { - let history = history - .into_iter() - .map(response::Tx::try_from) - .collect::, _>>() - .map_err(D::Error::custom)?; - spk_histories.insert_spk(spk_hash, status, history); - } - Ok(spk_histories) - } - } - - pub(super) mod anchors_as_seq { - use super::*; - use serde::{Deserialize, Deserializer, Serializer}; - - type Anchors = HashMap<(Txid, BlockHash), ProvenAnchor>; - - pub fn serialize( - anchors: &Anchors, - serializer: S, - ) -> Result { - serializer.collect_seq( - anchors - .iter() - .map(|(&(txid, block_hash), anchor)| (txid, block_hash, anchor)), - ) - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result { - Ok( - Vec::<(Txid, BlockHash, ProvenAnchor)>::deserialize(deserializer)? - .into_iter() - .map(|(txid, block_hash, anchor)| ((txid, block_hash), anchor)) - .collect(), - ) - } - } -} - #[cfg(test)] mod test { use super::*; - use bitcoin::hashes::Hash; + use bdk_core::bitcoin::{self, hashes::Hash}; fn txid(byte: u8) -> Txid { Txid::from_byte_array([byte; 32]) @@ -284,35 +233,25 @@ mod test { ElectrumScriptHash::from_byte_array([byte; 32]) } - /// `response::Tx` derives `Deserialize` only, so histories round-trip through `HistoryTx`. - /// Both of its variants have to survive the trip intact. - #[test] - fn spk_histories_round_trip() { - let history = vec![ - response::Tx::Confirmed(response::ConfirmedTx { - txid: txid(1), - height: bitcoin::absolute::Height::from_consensus(700_000).unwrap(), - }), - response::Tx::Mempool(response::MempoolTx { - txid: txid(2), - fee: bitcoin::Amount::from_sat(1234), - confirmed_inputs: false, - }), - ]; - let status = ElectrumScriptStatus::from_history(&history).expect("history is not empty"); - - let mut before = Subscriptions::default(); - before.insert_spk(spk_hash(9), status, history); - - let json = serde_json::to_string(&before).expect("must serialize"); - let after: Subscriptions = serde_json::from_str(&json).expect("must deserialize"); + /// A transaction whose txid varies with `unique`, so distinct calls yield distinct txids. + fn transaction(unique: u32) -> Transaction { + Transaction { + version: bitcoin::transaction::Version::ONE, + lock_time: bitcoin::absolute::LockTime::from_consensus(unique), + input: Vec::new(), + output: Vec::new(), + } + } - assert_eq!( - after.spk_history(status).map(<[_]>::len), - Some(2), - "the history must survive, and still answer to its status" - ); - assert_eq!(after.spk_status(spk_hash(9)), Some(status)); + fn anchor(height: u32, pos: usize) -> ProvenAnchor { + ProvenAnchor { + block_id: bdk_core::BlockId { + height, + hash: BlockHash::from_byte_array([height as u8; 32]), + }, + pos, + merkle: Vec::new(), + } } /// Two scripts paid by the same transaction, and nothing else, have identical histories — @@ -351,50 +290,157 @@ mod test { ); } - /// `anchors` is keyed by a tuple, which JSON cannot use as a map key. A caller who chooses - /// to persist a [`TxCache`] rather than rebuild it must still be able to. + /// A caller's wallet knows which spk each of its transactions paid, the transaction itself, + /// and whether it is confirmed (with its anchor) or still in the mempool — enough to + /// rebuild the transaction cache and compute the same status an Electrum server would, + /// without ever asking it. #[test] - fn tx_cache_round_trips_through_json() { - let anchor = (txid(1), bitcoin::BlockHash::from_byte_array([2; 32])); - let mut before = TxCache::default(); - before.anchors.insert( - anchor, - ProvenAnchor { - block_id: bdk_core::BlockId { - height: 2, - hash: bitcoin::BlockHash::from_byte_array([2; 32]), + fn from_wallet_txs_rebuilds_a_status_the_server_would_recognise() { + let spk = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_hash = ElectrumScriptHash::new(&spk); + let confirmed_tx = transaction(1); + let mempool_tx = transaction(2); + let confirmed_txid = confirmed_tx.compute_txid(); + let mempool_txid = mempool_tx.compute_txid(); + let confirmed_anchor = anchor(100, 0); + + let cache = Cache::from_wallet_txs([ + ( + confirmed_tx, + TxConfirmationStatus::Confirmed(confirmed_anchor.clone()), + vec![spk.clone()], + ), + ( + mempool_tx, + TxConfirmationStatus::Mempool { + confirmed_inputs: true, }, - pos: 0, - merkle: Vec::new(), - }, - ); + vec![spk], + ), + ]); - let json = serde_json::to_string(&before).expect("must serialize"); - let after: TxCache = serde_json::from_str(&json).expect("must deserialize"); + assert_eq!( + cache.spk_txids.get(&spk_hash).map(BTreeSet::len), + Some(2), + "both txids must be recorded against the spk" + ); + assert_eq!( + cache.txs.keys().copied().collect::>(), + BTreeSet::from([confirmed_txid, mempool_txid]), + "the transactions themselves must be cached too" + ); + assert_eq!( + cache + .anchors + .get(&(confirmed_txid, confirmed_anchor.block_id.hash)), + Some(&confirmed_anchor), + "the confirmed transaction's anchor must be cached, needing no reproof" + ); - assert_eq!(after.anchors.get(&anchor), before.anchors.get(&anchor)); + let expected = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: confirmed_txid, + height: bitcoin::absolute::Height::from_consensus(100).unwrap(), + }), + response::Tx::Mempool(response::MempoolTx { + txid: mempool_txid, + fee: bitcoin::Amount::ZERO, + confirmed_inputs: true, + }), + ]) + .expect("history is not empty"); + assert_eq!( + cache.subscriptions.spk_status(spk_hash), + Some(expected), + "the rebuilt status must match what a server hashing the same history would report" + ); } - /// A `Cache` carries none of it, so persisting one cannot go stale against the wallet. + /// The status hash itself only ever mixes in `txid:height:` — never a block position — so + /// two transactions confirmed in the same block only get the right status if `pos` orders + /// them the way the block does. Get it backwards and the computed status is simply wrong. #[test] - fn cache_does_not_persist_the_tx_cache() { - let mut before = Cache::default(); - before.tx_cache.txs.insert( - txid(1), - Arc::new(bitcoin::Transaction { - version: bitcoin::transaction::Version::ONE, - lock_time: bitcoin::absolute::LockTime::ZERO, - input: Vec::new(), - output: Vec::new(), + fn same_block_transactions_are_ordered_by_block_position() { + let spk = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_hash = ElectrumScriptHash::new(&spk); + let (first_tx, second_tx) = (transaction(1), transaction(2)); + let (first, second) = (first_tx.compute_txid(), second_tx.compute_txid()); + let height = bitcoin::absolute::Height::from_consensus(100).unwrap(); + + let cache = Cache::from_wallet_txs([ + ( + first_tx, + TxConfirmationStatus::Confirmed(anchor(100, 0)), + vec![spk.clone()], + ), + ( + second_tx, + TxConfirmationStatus::Confirmed(anchor(100, 1)), + vec![spk], + ), + ]); + + let in_block_order = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: first, + height, }), + response::Tx::Confirmed(response::ConfirmedTx { + txid: second, + height, + }), + ]) + .expect("history is not empty"); + let reversed = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: second, + height, + }), + response::Tx::Confirmed(response::ConfirmedTx { + txid: first, + height, + }), + ]) + .expect("history is not empty"); + assert_ne!( + in_block_order, reversed, + "the hash must actually be order-sensitive, or this test proves nothing" ); - let json = serde_json::to_string(&before).expect("must serialize"); - assert!( - !json.contains(&txid(1).to_string()), - "the wallet's own data must not be written here: {json}" + assert_eq!( + cache.subscriptions.spk_status(spk_hash), + Some(in_block_order), + "same-height entries must be ordered by their position in the block, not insertion \ + order, or the rebuilt status will not match what a server reports" ); - let after: Cache = serde_json::from_str(&json).expect("must deserialize"); - assert!(after.tx_cache.txs.is_empty()); + } + + /// A wallet holds each of its transactions once, not once per script it happens to pay — a + /// single entry naming every relevant script must still update every one of them. + #[test] + fn one_transaction_can_update_more_than_one_script() { + let spk_a = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_b = ScriptBuf::from_hex("0014000000000000000000000000000000000000000b").unwrap(); + let tx = transaction(1); + let txid = tx.compute_txid(); + + let cache = Cache::from_wallet_txs([( + tx, + TxConfirmationStatus::Confirmed(anchor(100, 0)), + vec![spk_a.clone(), spk_b.clone()], + )]); + + for spk in [spk_a, spk_b] { + let spk_hash = ElectrumScriptHash::new(&spk); + assert_eq!( + cache.spk_txids.get(&spk_hash), + Some(&BTreeSet::from([txid])), + "the tx must be recorded against every script it pays" + ); + assert!( + cache.subscriptions.spk_status(spk_hash).is_some(), + "and every script must get a status to subscribe with" + ); + } } } diff --git a/bdk_electrum_streaming/src/confirmation_job.rs b/bdk_electrum_streaming/src/confirmation_job.rs index ed33a55..7c85b9c 100644 --- a/bdk_electrum_streaming/src/confirmation_job.rs +++ b/bdk_electrum_streaming/src/confirmation_job.rs @@ -284,7 +284,7 @@ impl ConfirmationJob { continue; } }; - match cache.tx_cache.anchors.get(&(txid, header.block_hash())) { + match cache.anchors.get(&(txid, header.block_hash())) { Some(anchor) => { resolved.insert((anchor.clone(), txid)); } diff --git a/bdk_electrum_streaming/src/spk_job.rs b/bdk_electrum_streaming/src/spk_job.rs index b9ff34f..37da75b 100644 --- a/bdk_electrum_streaming/src/spk_job.rs +++ b/bdk_electrum_streaming/src/spk_job.rs @@ -97,7 +97,7 @@ impl SpkJob { let stage = match spk_status { Some(status) => SpkStage::ProcessingHistory { status }, None => { - if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&spk_hash) { + if let Some(prev_txids) = cache.spk_txids.get(&spk_hash) { tx_update .evicted_ats .extend(prev_txids.iter().map(|&txid| (txid, start.as_secs()))); @@ -151,7 +151,7 @@ impl SpkJob { SpkStage::ProcessingHistory { status } => { match cache.subscriptions.spk_history(*status) { Some(history) => { - if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&self.spk_hash) { + if let Some(prev_txids) = cache.spk_txids.get(&self.spk_hash) { let these_txids = history.iter().map(|tx| tx.txid()).collect::>(); let to_evict = prev_txids @@ -178,7 +178,7 @@ impl SpkJob { } } SpkStage::ProcessingTxs(missing_txs) => { - missing_txs.retain(|txid| match cache.tx_cache.txs.get(txid) { + missing_txs.retain(|txid| match cache.txs.get(txid) { Some(tx) => { self.tx_update.txs.push(tx.clone()); false @@ -207,7 +207,7 @@ impl SpkJob { // `retain` cannot fail, so a bad output is carried out and raised below. let mut err = Option::::None; missing_prevouts.retain(|op| { - let tx = match cache.tx_cache.txs.get(&op.txid) { + let tx = match cache.txs.get(&op.txid) { Some(tx) => tx, None => { let txid = op.txid; diff --git a/bdk_electrum_streaming/src/state.rs b/bdk_electrum_streaming/src/state.rs index d99e302..5bdb389 100644 --- a/bdk_electrum_streaming/src/state.rs +++ b/bdk_electrum_streaming/src/state.rs @@ -258,7 +258,6 @@ impl State { let resp_status = ElectrumScriptStatus::from_history(&resp); if let Some(spk_status) = resp_status { self.cache - .tx_cache .spk_txids .entry(req.script_hash) .or_default() @@ -300,7 +299,7 @@ impl State { txid, )); } - self.cache.tx_cache.txs.insert(get_tx.txid, resp.tx.into()); + self.cache.txs.insert(get_tx.txid, resp.tx.into()); self.poll_spk_jobs(req_queue, job_ids)?; self.poll_confirmation_job(req_queue) } @@ -343,7 +342,7 @@ impl State { block_hash = header.block_hash().to_string(), "Inserting anchor.", ); - self.cache.tx_cache.anchors.insert( + self.cache.anchors.insert( (req.txid, header.block_hash()), ProvenAnchor { block_id: BlockId { @@ -382,7 +381,12 @@ impl State { } /// React to the server announcing `header` at `height` as its tip. - fn on_new_tip(&mut self, req_queue: &mut ReqQueue, height: u32, header: Header) -> anyhow::Result<()> { + fn on_new_tip( + &mut self, + req_queue: &mut ReqQueue, + height: u32, + header: Header, + ) -> anyhow::Result<()> { match &mut self.confirmation_job { Some(job) => { if job.set_tip(height, header) { @@ -416,7 +420,7 @@ impl State { self.cache.subscriptions.remove_spk(spk_hash); } - if spk_status.is_some() || self.cache.tx_cache.spk_txids.contains_key(&spk_hash) { + if spk_status.is_some() || self.cache.spk_txids.contains_key(&spk_hash) { for script_hash in self.spk_tracker.mark_script_hash_used(&k, i) { self.coord .queuer(req_queue, JobId::Spk(script_hash)) diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index 25824c7..9150c75 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -337,7 +337,7 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( let header_2 = block_with_tx(&header_1, txid, 200, 0); let mut cache = Cache::default(); - cache.tx_cache.txs.insert(txid, Arc::new(tx.clone())); + cache.txs.insert(txid, Arc::new(tx.clone())); let mut state = new_state(cache, descriptor); let mut queue = ReqQueue::new(); @@ -961,7 +961,7 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { "the job must give up on the pair rather than re-ask" ); assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "an error proves nothing, so no anchor may be recorded from it" ); @@ -1603,7 +1603,7 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { state.poll(&mut queue, response(&req, &server))?; } assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "a proof for a block we do not have must not anchor anything" ); @@ -1682,7 +1682,7 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( state.poll(&mut queue, resp)?; } assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "an error must not anchor anything" ); assert_eq!( @@ -2076,7 +2076,7 @@ fn an_anchor_above_the_announced_tip_does_not_spin() -> anyhow::Result<()> { "no tip was announced, so the chain must not have moved" ); assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "and nothing may be anchored at a height the chain has not reached" ); From 66d31fa647e732c2e9ea33e35e9a5eb65b6e8f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 2 Sep 2026 03:50:16 +0000 Subject: [PATCH 4/9] feat: Add trusted-headers-gen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HeaderChain has to be handed trusted headers, and trusting one means asserting it is canonical — which has to come from somewhere the operator actually checked, ideally their own node. `cargo run -p trusted-headers-gen -- --url ... --network bitcoin` fetches a header from Bitcoin Core and writes a Rust module exposing a `..._TRUSTED_HEADERS` per network plus a `trusted_headers()` map over them. A node serves one network, so each run targets one and only touches that network's section of the output, merging into whatever heights are already recorded for the others. Nothing is shipped by this repo: the output lands in the current directory, and placing it takes an explicit `--out`, so the data lives in the tree of whoever reviewed it as the diff it is. The default height is the second highest difficulty-adjustment boundary at or below the tip: a boundary, because a header-verifying client needs its highest trusted header on one to recompute every retarget above it, and the second one because the highest can be the tip itself — a block a handful of confirmations deep is no basis for trust. `--network` is checked against what the node reports for `chain`, since a header filed under the wrong network is a poisoned anchor and the node is the one thing that knows which chain it serves. That field is read out of the raw response rather than a typed one: the rest of `getblockchaininfo` has changed shape across Core releases, and this uses the v17 client surface precisely to stay version-agnostic. Tested against Bitcoin Core v28.1: a test spins up a real bitcoind and checks the header fetched is the one that node has at that height, that a height above the tip is refused, that a regtest node cannot be talked into answering for mainnet, and that the result survives the round trip out into generated source and back. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013Ga32WdTURWnpMDxAckM2K --- Cargo.lock | 11 + Cargo.toml | 2 +- trusted-headers-gen/Cargo.toml | 17 + trusted-headers-gen/src/main.rs | 553 ++++++++++++++++++++++++++++++++ 4 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 trusted-headers-gen/Cargo.toml create mode 100644 trusted-headers-gen/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index f92e793..e01fcb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,6 +1053,17 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trusted-headers-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "bdk_core", + "bdk_testenv", + "corepc-client", + "serde_json", +] + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 3649617..6eeee81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] resolver = "2" -members = ["bdk_electrum_streaming"] +members = ["bdk_electrum_streaming", "trusted-headers-gen"] diff --git a/trusted-headers-gen/Cargo.toml b/trusted-headers-gen/Cargo.toml new file mode 100644 index 0000000..ca494c6 --- /dev/null +++ b/trusted-headers-gen/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "trusted-headers-gen" +version = "0.1.0" +description = "Fetches a trusted header from Bitcoin Core and writes it out as reviewable Rust source" +license = "MIT OR Apache-2.0" +edition = "2021" +rust-version = "1.85" +repository = "https://github.com/evanlinjin/experiments" + +[dependencies] +anyhow = "1" +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +corepc-client = { version = "0.13", features = ["client-sync"] } +serde_json = "1" + +[dev-dependencies] +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } diff --git a/trusted-headers-gen/src/main.rs b/trusted-headers-gen/src/main.rs new file mode 100644 index 0000000..7b6a57d --- /dev/null +++ b/trusted-headers-gen/src/main.rs @@ -0,0 +1,553 @@ +//! Fetches a header from a Bitcoin Core node and writes it to `./trusted_headers.rs` (or +//! wherever `--out` says), for whichever network that node is running. +//! +//! Bitcoin Core only ever serves one network per RPC endpoint, so this is meant to be run once +//! per network you want a trusted header for, each time pointed at a node on that network. Every +//! run only touches its own network's section of the output file, adding a new height or +//! replacing an existing one — the other networks' sections are left exactly as they were. +//! +//! What comes out is a Rust module to vendor into whichever crate needs it, exposing a +//! `..._TRUSTED_HEADERS` per network plus a `trusted_headers()` map over them. Nothing ships +//! this data for you: trusting a header means asserting it is canonical, so it belongs in the +//! tree of whoever checked it, having been reviewed as the diff it is. That is also why the +//! output lands in the current directory by default and placing it takes an explicit `--out`. +//! +//! ```text +//! cargo run -p trusted-headers-gen -- \ +//! --url http://127.0.0.1:8332 \ +//! --cookie ~/.bitcoin/.cookie \ +//! --network bitcoin \ +//! [--height 903168] \ +//! [--out trusted_headers.rs] +//! ``` +//! +//! `--height` defaults to the *second* highest difficulty-adjustment boundary (a multiple of +//! 2016) at or below the node's tip — a boundary, because a header-verifying client needs its +//! highest trusted header on one to recompute every retarget above it, and the second one +//! because the highest can sit right at the tip. A lower, explicit `--height` is for backfill: +//! a trusted block below where syncing starts needs no such alignment. + +use std::{ + collections::BTreeMap, + fmt::Write as _, + fs, + path::{Path, PathBuf}, + str::FromStr, +}; + +use bdk_core::bitcoin::{block::Header, consensus::encode::deserialize_hex, BlockHash, Network}; +use corepc_client::client_sync::{v17::Client, Auth}; + +/// Per-network state: trusted heights and the raw header hex at each. Rebuilt from the existing +/// output file (if any) on every run, so a run for one network cannot lose another's entries. +type State = BTreeMap>; + +const ALL_NETWORKS: [Network; 5] = [ + Network::Bitcoin, + Network::Testnet, + Network::Testnet4, + Network::Signet, + Network::Regtest, +]; + +/// The prefix this network's generated consts carry, e.g. `Network::Bitcoin` -> `MAINNET`. +fn const_prefix(network: Network) -> &'static str { + match network { + Network::Bitcoin => "MAINNET", + Network::Testnet => "TESTNET", + Network::Testnet4 => "TESTNET4", + Network::Signet => "SIGNET", + Network::Regtest => "REGTEST", + } +} + +/// What `getblockchaininfo` calls this network, as defined by BIP70. +fn chain_name(network: Network) -> &'static str { + match network { + Network::Bitcoin => "main", + Network::Testnet => "test", + Network::Testnet4 => "testnet4", + Network::Signet => "signet", + Network::Regtest => "regtest", + } +} + +struct Args { + url: String, + cookie: Option, + user: Option, + pass: Option, + network: Network, + height: Option, + out: PathBuf, +} + +impl Args { + fn parse() -> anyhow::Result { + let mut url = None; + let mut cookie = None; + let mut user = None; + let mut pass = None; + let mut network = None; + let mut height = None; + let mut out = None; + + let mut args = std::env::args().skip(1); + while let Some(flag) = args.next() { + let mut value = || { + args.next() + .ok_or_else(|| anyhow::anyhow!("{flag} needs a value")) + }; + match flag.as_str() { + "--url" => url = Some(value()?), + "--cookie" => cookie = Some(PathBuf::from(value()?)), + "--user" => user = Some(value()?), + "--pass" => pass = Some(value()?), + "--network" => { + network = Some(match value()?.as_str() { + "bitcoin" | "mainnet" => Network::Bitcoin, + "testnet" | "testnet3" => Network::Testnet, + "testnet4" => Network::Testnet4, + "signet" => Network::Signet, + "regtest" => Network::Regtest, + other => anyhow::bail!("unknown --network {other}"), + }) + } + "--height" => height = Some(value()?.parse()?), + "--out" => out = Some(PathBuf::from(value()?)), + other => anyhow::bail!("unknown flag {other}"), + } + } + + Ok(Self { + url: url.ok_or_else(|| anyhow::anyhow!("--url is required"))?, + cookie, + user, + pass, + network: network.ok_or_else(|| anyhow::anyhow!("--network is required"))?, + height, + out: out.unwrap_or_else(|| PathBuf::from("trusted_headers.rs")), + }) + } +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse()?; + + let auth = match (&args.cookie, &args.user, &args.pass) { + (Some(cookie), _, _) => Auth::CookieFile(cookie.clone()), + (None, Some(user), Some(pass)) => Auth::UserPass(user.clone(), pass.clone()), + _ => Auth::None, + }; + let client = if matches!(auth, Auth::None) { + Client::new(&args.url) + } else { + Client::new_with_auth(&args.url, auth).map_err(|e| anyhow::anyhow!(e.to_string()))? + }; + + let (height, hash, header_hex) = fetch_trusted_header(&client, args.network, args.height)?; + + let mut state = parse_existing(&args.out); + state + .entry(args.network) + .or_default() + .insert(height, header_hex); + fs::write(&args.out, render_file(&state))?; + let _ = std::process::Command::new("rustfmt") + .arg(&args.out) + .status(); + + println!( + "{:?}: trusted header at height {height} ({hash}) written to {}", + args.network, + args.out.display(), + ); + Ok(()) +} + +/// Blocks between difficulty adjustments. +const RETARGET_INTERVAL: u32 = 2016; + +/// The height to trust when the caller does not name one: the *second* highest +/// difficulty-adjustment boundary at or below `tip`. +/// +/// A boundary, because the highest trusted block has to sit on one for every retarget above it +/// to be recomputable rather than taken on faith. The second one rather than the highest, +/// because the highest can be the tip itself — and a block a handful of confirmations deep is +/// no basis for trust. Stepping back a whole retarget period puts the anchor between 2016 and +/// 4032 blocks behind the tip, deep enough that reorging past it is not a live concern. +/// +/// Chains too short for that (a fresh regtest, say) fall back to genesis. +fn default_trusted_height(tip: u32) -> u32 { + (tip - tip % RETARGET_INTERVAL).saturating_sub(RETARGET_INTERVAL) +} + +/// Ask the node for the header to trust, returning its height, hash and raw hex. +/// +/// `height` defaults to [`default_trusted_height`]. The node is held to `network` first: a +/// header filed under the wrong network is a poisoned trust anchor, and the node is the one +/// thing that actually knows which chain it is serving. The header is then decoded and checked +/// to hash to the block the node named, so nothing further downstream has to. +fn fetch_trusted_header( + client: &Client, + network: Network, + height: Option, +) -> anyhow::Result<(u32, BlockHash, String)> { + // Read `chain` out of the raw response rather than through a typed one: the shape of the + // rest of `getblockchaininfo` has changed across Core releases, and `chain` is the only + // field here that matters. + let info: serde_json::Value = client + .call("getblockchaininfo", &[]) + .map_err(|e| anyhow::anyhow!("getblockchaininfo: {e}"))?; + let chain = info + .get("chain") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| anyhow::anyhow!("getblockchaininfo gave no `chain`"))?; + anyhow::ensure!( + chain == chain_name(network), + "node is on {chain}, but --network says {network:?} ({})", + chain_name(network), + ); + + let tip = client + .get_block_count() + .map_err(|e| anyhow::anyhow!("getblockcount: {e}"))? + .0 as u32; + let height = height.unwrap_or_else(|| default_trusted_height(tip)); + anyhow::ensure!( + height <= tip, + "height {height} is above the node's tip {tip}" + ); + + let hash = client + .get_block_hash(height as u64) + .map_err(|e| anyhow::anyhow!("getblockhash({height}): {e}"))? + .0; + let hash = + BlockHash::from_str(&hash).map_err(|e| anyhow::anyhow!("server gave a bad hash: {e}"))?; + let header_hex = client + .get_block_header(&hash) + .map_err(|e| anyhow::anyhow!("getblockheader({hash}): {e}"))? + .0; + // Checked once here, so the generated file never needs to check it again. + let header: Header = deserialize_hex(&header_hex)?; + anyhow::ensure!( + header.block_hash() == hash, + "server gave a header for a different block than the hash it named" + ); + Ok((height, hash, header_hex)) +} + +/// Read back whatever `(height, hex)` pairs the file already has, per network — empty if the +/// file does not exist yet, or has nothing for a given network. +fn parse_existing(path: &Path) -> State { + let mut state = State::new(); + let Ok(text) = fs::read_to_string(path) else { + return state; + }; + for network in ALL_NETWORKS { + let prefix = const_prefix(network); + let Some(body) = section_body(&text, prefix) else { + continue; + }; + let mut heights = BTreeMap::new(); + for line in body.lines() { + if let Some((height, hex)) = parse_entry_line(line) { + heights.insert(height, hex); + } + } + if !heights.is_empty() { + state.insert(network, heights); + } + } + state +} + +fn section_body<'a>(text: &'a str, prefix: &str) -> Option<&'a str> { + let begin = format!("// --- BEGIN {prefix} ---"); + let end = format!("// --- END {prefix} ---"); + text.split(&begin).nth(1)?.split(&end).next() +} + +/// Parse a rendered `(height, "hex"),` entry line — the only shape of line this tool ever +/// writes inside a section body. +fn parse_entry_line(line: &str) -> Option<(u32, String)> { + let rest = line.trim().strip_prefix('(')?; + let (height, rest) = rest.split_once(',')?; + let height = height.trim().parse().ok()?; + let hex = rest.trim().strip_prefix('"')?; + let hex = hex.split('"').next()?; + Some((height, hex.to_string())) +} + +/// Render the whole module. `state` is never empty — a run always has a header to add — so +/// every import and helper written here is used by at least one section below. +fn render_file(state: &State) -> String { + let mut out = String::new(); + let w = &mut out; + writeln!( + w, + "// @generated by `cargo run -p trusted-headers-gen`.\n\ + // Do not hand-edit — rerun the generator instead; each run only touches the network\n\ + // it points at." + ) + .unwrap(); + writeln!(w).unwrap(); + writeln!(w, "use std::{{collections::BTreeMap, sync::LazyLock}};").unwrap(); + writeln!(w).unwrap(); + writeln!( + w, + "use bdk_core::bitcoin::{{block::Header, consensus::encode::deserialize_hex, Network}};" + ) + .unwrap(); + writeln!(w).unwrap(); + writeln!( + w, + "/// Deserialize each `(height, hex)` pair — checked valid when this file was generated." + ) + .unwrap(); + writeln!( + w, + "fn parse(raw: [(u32, &str); N]) -> [(u32, Header); N] {{" + ) + .unwrap(); + writeln!( + w, + " raw.map(|(height, hex)| (height, deserialize_hex(hex).expect(\"checked at generation time\")))" + ) + .unwrap(); + writeln!(w, "}}").unwrap(); + writeln!(w).unwrap(); + + for network in ALL_NETWORKS { + let Some(headers) = state.get(&network) else { + continue; + }; + let prefix = const_prefix(network); + let n = headers.len(); + writeln!(w, "// --- BEGIN {prefix} ---").unwrap(); + writeln!( + w, + "const {prefix}_TRUSTED_HEADERS_HEX: [(u32, &str); {n}] = [" + ) + .unwrap(); + for (height, hex) in headers { + writeln!(w, " ({height}, \"{hex}\"),").unwrap(); + } + writeln!(w, "];").unwrap(); + writeln!(w).unwrap(); + writeln!(w, "/// Trusted headers for `Network::{network:?}`.").unwrap(); + writeln!( + w, + "pub static {prefix}_TRUSTED_HEADERS: LazyLock<[(u32, Header); {n}]> =" + ) + .unwrap(); + writeln!( + w, + " LazyLock::new(|| parse({prefix}_TRUSTED_HEADERS_HEX));" + ) + .unwrap(); + writeln!(w, "// --- END {prefix} ---").unwrap(); + writeln!(w).unwrap(); + } + + writeln!( + w, + "/// Every network there are trusted headers here for, keyed by [`Network`]." + ) + .unwrap(); + writeln!( + w, + "pub fn trusted_headers() -> BTreeMap {{" + ) + .unwrap(); + writeln!(w, " let mut map = BTreeMap::new();").unwrap(); + for network in ALL_NETWORKS { + if !state.contains_key(&network) { + continue; + } + let prefix = const_prefix(network); + writeln!( + w, + " map.insert(Network::{network:?}, {prefix}_TRUSTED_HEADERS.as_slice());" + ) + .unwrap(); + } + writeln!(w, " map").unwrap(); + writeln!(w, "}}").unwrap(); + + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_state() -> State { + let mut state = State::new(); + state.insert( + Network::Bitcoin, + BTreeMap::from([(902016, "aa".repeat(80))]), + ); + state.insert( + Network::Signet, + BTreeMap::from([(0, "bb".repeat(80)), (2016, "cc".repeat(80))]), + ); + state + } + + /// The default height must always land on a retarget boundary, and always a whole period + /// behind the tip — never the boundary the tip itself may be sitting on. + #[test] + fn default_height_steps_back_a_whole_retarget_period() { + // 2016 * 450 == 907_200, so that is a boundary and 905_184 is the one below it. + assert_eq!( + default_trusted_height(907_200), + 905_184, + "tip on a boundary" + ); + assert_eq!(default_trusted_height(907_201), 905_184, "just past one"); + assert_eq!(default_trusted_height(907_199), 903_168, "just before one"); + + for tip in [2016_u32, 4032, 100_000, 907_201, 1_000_000] { + let height = default_trusted_height(tip); + assert_eq!(height % RETARGET_INTERVAL, 0, "{height} is not a boundary"); + let depth = tip - height; + assert!( + (RETARGET_INTERVAL..=RETARGET_INTERVAL * 2).contains(&depth), + "tip {tip} put the anchor {depth} blocks deep", + ); + } + + // Chains with no such boundary behind them fall back to genesis. + for tip in [0, 1, 5, 2015] { + assert_eq!(default_trusted_height(tip), 0, "short chain at tip {tip}"); + } + } + + /// The RPC path itself, against a real `bitcoind`: what the tool fetches has to be the + /// header that node actually has at that height, and it has to survive the trip out into + /// generated source and back. + /// + /// Needs a `bitcoind` executable — `BITCOIND_EXE`, a downloaded one, or one on `PATH`. + /// Skipped when there is none, since that is a missing environment rather than a broken + /// tool. + #[test] + fn fetches_a_real_header_from_bitcoin_core() -> anyhow::Result<()> { + let Ok(exe) = bdk_testenv::bitcoind::exe_path() else { + eprintln!("skipping: no bitcoind executable found"); + return Ok(()); + }; + let node = bdk_testenv::bitcoind::BitcoinD::new(exe)?; + let address = node.client.new_address()?; + node.client.generate_to_address(5, &address)?; + + let client = Client::new_with_auth( + &node.rpc_url(), + Auth::CookieFile(node.params.cookie_file.clone()), + ) + .map_err(|e| anyhow::anyhow!(e.to_string()))?; + + // An explicit height, held to the hash the node itself reports for it. + let (height, hash, hex) = fetch_trusted_header(&client, Network::Regtest, Some(3))?; + assert_eq!(height, 3); + assert_eq!(hash, node.client.get_block_hash(3)?.block_hash()?); + assert_eq!(hex.len(), 160, "a header is 80 bytes, so 160 hex chars"); + + // No height given: this chain is far too short to have a whole retarget period behind + // a boundary, so it falls back to genesis. + let (default_height, genesis, _) = fetch_trusted_header(&client, Network::Regtest, None)?; + assert_eq!(default_height, 0); + assert_eq!(genesis, node.client.get_block_hash(0)?.block_hash()?); + + // A height the node cannot answer for is refused, not quietly turned into something. + assert!(fetch_trusted_header(&client, Network::Regtest, Some(500)).is_err()); + + // A regtest node cannot be talked into supplying a header filed under mainnet: the + // node knows which chain it serves, and mislabelling one poisons the trust anchor. + let err = fetch_trusted_header(&client, Network::Bitcoin, Some(3)) + .expect_err("a regtest node must not answer for mainnet") + .to_string(); + assert!(err.contains("node is on regtest"), "{err}"); + + // What came back renders into source that parses back to the very same hex. + let mut state = State::new(); + state + .entry(Network::Regtest) + .or_default() + .insert(height, hex.clone()); + let file = tempfile(); + fs::write(&file, render_file(&state))?; + assert_eq!(parse_existing(&file)[&Network::Regtest][&height], hex); + let _ = fs::remove_file(&file); + Ok(()) + } + + #[test] + fn render_then_parse_round_trips() { + let state = sample_state(); + let rendered = render_file(&state); + let file = tempfile(); + fs::write(&file, &rendered).unwrap(); + assert_eq!(parse_existing(&file), state); + let _ = fs::remove_file(&file); + } + + /// Regenerating for one network must not disturb another's entries, or a run for testnet + /// would wipe out mainnet's trusted header. + #[test] + fn updating_one_network_preserves_the_others() { + let mut state = sample_state(); + let rendered = render_file(&state); + let file = tempfile(); + fs::write(&file, &rendered).unwrap(); + + let mut reloaded = parse_existing(&file); + reloaded + .entry(Network::Testnet) + .or_default() + .insert(4032, "dd".repeat(80)); + fs::write(&file, render_file(&reloaded)).unwrap(); + + state + .entry(Network::Testnet) + .or_default() + .insert(4032, "dd".repeat(80)); + assert_eq!(parse_existing(&file), state); + let _ = fs::remove_file(&file); + } + + /// A fresh height for a network already in the file adds to it; the same height replaces. + #[test] + fn same_height_replaces_new_height_adds() { + let state = sample_state(); + let file = tempfile(); + fs::write(&file, render_file(&state)).unwrap(); + + let mut reloaded = parse_existing(&file); + reloaded + .get_mut(&Network::Bitcoin) + .unwrap() + .insert(902016, "ee".repeat(80)); // replace + reloaded + .get_mut(&Network::Bitcoin) + .unwrap() + .insert(904032, "ff".repeat(80)); // add + fs::write(&file, render_file(&reloaded)).unwrap(); + + let mainnet = &parse_existing(&file)[&Network::Bitcoin]; + assert_eq!(mainnet.get(&902016), Some(&"ee".repeat(80))); + assert_eq!(mainnet.get(&904032), Some(&"ff".repeat(80))); + assert_eq!(mainnet.len(), 2); + + let _ = fs::remove_file(&file); + } + + fn tempfile() -> PathBuf { + std::env::temp_dir().join(format!( + "trusted_headers_gen_test_{:?}_{}", + std::thread::current().id(), + std::process::id(), + )) + } +} From 875f0239281790a88529369c4816a7b4511823eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 2 Sep 2026 04:17:16 +0000 Subject: [PATCH 5/9] fix(bdk_electrum_streaming): Close two holes found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `HeaderChain::apply` took the backfill path for any run starting below the base, requiring only that it reach *at least* the base. A run that also carried on past the tip took that path while replacing verified blocks — and the backfill path rebuilds the chain rather than comparing work, so an equal-work fork was accepted there, which the extend path exists to refuse. Not reachable through `ConfirmationJob`, whose backfills always stop at `base - 1`, but `apply` is public and its contract read as a lower bound. A backfill must now fill the gap exactly. `ConfirmationJob` planned no run at all for a target more than `REORG_WINDOW` below the verified tip, then found its remaining runs complete, applied nothing, and handed back the chain it already had as though the server had confirmed it — while going on to ask that server for proofs at heights it does not have. The `REORG_WINDOW` doc claimed the connection errors out in this case; now it does. Checked only once there is a verified tip: before that, planning nothing is an ordinary waypoint, since a backfill is planned from heights the histories have yet to name. Also re-queue the gaps when a `FetchHeaders` run is still incomplete. A server may answer `blockchain.block.headers` with fewer headers than asked for, and nothing else would ask again — `Init` is only re-entered when the target or the statuses move, which on a settled chain may be never. Requests in flight are deduplicated, so this cannot pile up. Both chain fixes come with a regression test that fails without them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013Ga32WdTURWnpMDxAckM2K --- .../src/confirmation_job.rs | 35 ++++++++++ bdk_electrum_streaming/src/header_chain.rs | 42 ++++++++++-- bdk_electrum_streaming/tests/state.rs | 65 +++++++++++++++++++ 3 files changed, 137 insertions(+), 5 deletions(-) diff --git a/bdk_electrum_streaming/src/confirmation_job.rs b/bdk_electrum_streaming/src/confirmation_job.rs index 7c85b9c..c704e14 100644 --- a/bdk_electrum_streaming/src/confirmation_job.rs +++ b/bdk_electrum_streaming/src/confirmation_job.rs @@ -205,6 +205,7 @@ impl ConfirmationJob { ) -> anyhow::Result { match core::mem::take(&mut self.stage) { ConfirmationStage::Init => { + self.ensure_target_is_reachable(chain)?; let runs = self.required_runs(cache, chain); for (&start, &end) in &runs { self.queue_gaps(queuer, start, end); @@ -217,6 +218,14 @@ impl ConfirmationJob { (start..=end).all(|h| self.fetched_headers.contains_key(&h)) }); if !complete { + // Ask again for whatever is still missing. A server may answer + // `blockchain.block.headers` with fewer headers than were asked for, and + // nothing else would re-queue them: `Init` is only re-entered when the + // target or the statuses move, which on a settled chain may be never. + // Requests still in flight are deduplicated, so this cannot pile up. + for (&start, &end) in &runs { + self.queue_gaps(queuer, start, end); + } self.stage = ConfirmationStage::FetchHeaders { runs }; return Ok(ConfirmationProgress::Blocked); } @@ -321,6 +330,32 @@ impl ConfirmationJob { } } + /// Refuse a target too far below the verified tip to plan a run to. + /// + /// [`Self::required_runs`] only reaches back a [`REORG_WINDOW`](Self::REORG_WINDOW) below + /// the verified tip, so a target under that plans no run to it at all. Left alone the job + /// would find its remaining runs complete, apply nothing, and hand the caller back the + /// chain it already had as though the server had just confirmed it — then go on asking that + /// server for proofs at heights it does not have. A server that far behind cannot answer + /// for this wallet, whether it is mid-sync, was rolled back, or is one of several behind + /// one endpoint at different heights. + /// + /// Only checked once there *is* a verified tip. Before that there is no window to fall + /// below, and planning nothing is an ordinary waypoint rather than a fault: a target at or + /// under the block the chain starts at plans its backfill from the heights a history names, + /// which the histories have yet to arrive to name. + fn ensure_target_is_reachable(&self, chain: &HeaderChain) -> anyhow::Result<()> { + if let Some(tip) = chain.tip_height() { + anyhow::ensure!( + self.target_height >= tip.saturating_sub(Self::REORG_WINDOW), + "server's tip is {}, more than {} blocks below the verified tip {tip}", + self.target_height, + Self::REORG_WINDOW, + ); + } + Ok(()) + } + /// The heights carrying a transaction we have to anchor. fn anchor_heights(&self, cache: &Cache) -> BTreeSet<(u32, Txid)> { cache diff --git a/bdk_electrum_streaming/src/header_chain.rs b/bdk_electrum_streaming/src/header_chain.rs index c249902..8bc4843 100644 --- a/bdk_electrum_streaming/src/header_chain.rs +++ b/bdk_electrum_streaming/src/header_chain.rs @@ -136,8 +136,14 @@ impl HeaderChain { /// Apply a contiguous, ascending run of `headers` beginning at `start`. /// /// The run may extend the tip, replace it (reorg), or sit below the current - /// [`base_height`](Self::base_height) to backfill history — in which case it must reach up to - /// the existing base. + /// [`base_height`](Self::base_height) to backfill history — in which case it must stop + /// exactly where the chain already begins, filling the gap and no more. + /// + /// A backfill is rebuilt rather than compared for work, because the blocks it adds sit + /// below everything already verified and displace nothing. A run that started below the + /// base and carried on past the tip would take that path while replacing verified blocks, + /// so it is refused: reorging the tip is the extend path's business, where the replacement + /// has to out-work what it replaces. /// /// The chain is left untouched if anything fails to verify. pub fn apply(&mut self, start: u32, headers: Vec
) -> anyhow::Result<()> { @@ -160,10 +166,13 @@ impl HeaderChain { let cp = match self.cp.clone() { // Backfill: rebuild as the trusted blocks, the run, then whatever sat above the run. Some(cp) if start < self.base => { + // Exactly, not merely far enough: a run that carried on past the base would + // replace verified blocks here, where nothing compares work. ensure!( - end + 1 >= self.base, - "backfilled headers stop at {end}, below the chain base {}", - self.base + end + 1 == self.base, + "a backfill must stop where the chain begins, at {}, but these headers \ + stop at {end}", + self.base - 1, ); let above = cp .iter() @@ -550,6 +559,29 @@ mod test { assert!(c.apply(4, headers[4..7].to_vec()).is_err()); } + /// A run starting below the base takes the backfill path, which rebuilds the chain rather + /// than comparing work — so a run that also reaches past the tip could swap out verified + /// blocks for an equal-work fork, the very thing + /// [`rejects_a_reorg_with_less_work`](Self::rejects_a_reorg_with_less_work) forbids on the + /// extend path. A backfill has to fill the gap and stop. + #[test] + fn rejects_backfill_that_overshoots_the_base() { + let headers = mine(¶ms(), 10); + let mut c = chain(&headers, &[3]); + c.apply(4, headers[4..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(10)); + + // Real history up to the trusted block, then a fork of equal length — and so equal + // work — over everything above it. + let forked = fork(¶ms(), &headers, 3, 7, None); + let run = headers[1..=3].iter().copied().chain(forked).collect(); + + let err = c.apply(1, run).unwrap_err().to_string(); + assert!(err.contains("backfill"), "{err}"); + assert_eq!(c.header(5), Some(headers[5]), "chain is left untouched"); + assert_eq!(c.tip_height(), Some(10)); + } + #[test] fn rejects_a_trusted_anchor_off_the_retarget_boundary() { let params = retarget_params(); diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index 9150c75..8b77505 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -2299,3 +2299,68 @@ fn a_transaction_that_is_not_the_one_asked_for_is_rejected() -> anyhow::Result<( ); Ok(()) } + +/// A server announcing a tip far below the one we have already verified cannot be synced from: +/// no run is ever planned back that far, so the job would find nothing to do, apply nothing, +/// and hand back the chain we already had as though that server had just confirmed it — then +/// go on asking it for proofs at heights it does not have. +/// +/// This is what an endpoint load-balancing across nodes at different heights looks like, or a +/// node that was rolled back under us. +#[test] +fn a_tip_far_below_the_verified_one_stops_the_connection() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut chain = vec![genesis, header_1, header_2]; + for height in 3..=30u32 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + 0, + )); + } + + let mut state = new_state(Cache::default(), descriptor); + let mut queue = ReqQueue::new(); + let server = Server { + headers: chain.clone(), + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.start(&mut queue); + drain_requests(&mut state, &mut queue, &server); + assert_eq!( + state.chain().tip_height(), + Some(30), + "the chain must be verified up to the tip first" + ); + + // The server now claims a tip 25 blocks below the one we hold — past the window any run + // reaches back to. + let err = state + .poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&chain[5]), "height": 5 }], + })), + ) + .expect_err("a tip that far below the verified one must not be accepted"); + let msg = format!("{err:#}"); + assert!(msg.contains("below the verified tip"), "{msg}"); + + assert_eq!( + state.chain().tip_height(), + Some(30), + "and the verified chain must be left as it was" + ); + Ok(()) +} From 2dc02eb3003c56e753a3f675575fbac5717bbf88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 2 Sep 2026 04:17:17 +0000 Subject: [PATCH 6/9] chore: Pin the bdk git dependencies to a rev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `branch = "master"` re-resolves on every fresh checkout, so an upstream change could break or, worse, quietly alter this crate between builds — not what you want underneath proof-of-work verification. Pin the rev the lock file already resolved to. These stay git dependencies until `CheckPoint` and `FullScanResponse` are released; `cargo publish` rejects git dependencies outright, so that release has to wait for them regardless. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013Ga32WdTURWnpMDxAckM2K --- Cargo.lock | 6 +++--- bdk_electrum_streaming/Cargo.toml | 6 +++--- trusted-headers-gen/Cargo.toml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e01fcb5..e1a5c0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,7 +62,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bdk_chain" version = "0.23.2" -source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_core", "bitcoin", @@ -72,7 +72,7 @@ dependencies = [ [[package]] name = "bdk_core" version = "0.6.2" -source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bitcoin", "hashbrown", @@ -102,7 +102,7 @@ dependencies = [ [[package]] name = "bdk_testenv" version = "0.13.1" -source = "git+https://github.com/bitcoindevkit/bdk.git?branch=master#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_chain", "bitcoin", diff --git a/bdk_electrum_streaming/Cargo.toml b/bdk_electrum_streaming/Cargo.toml index 20759a2..1b61f85 100644 --- a/bdk_electrum_streaming/Cargo.toml +++ b/bdk_electrum_streaming/Cargo.toml @@ -13,8 +13,8 @@ readme = "README.md" futures = "0.3" futures-timer = "3" anyhow = "1" -bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master", features = ["serde"] } -bdk_chain = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb", features = ["serde"] } +bdk_chain = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } miniscript = { version = "13.0.0" } electrum_streaming_client = { version = "0.4" } serde = { version = "1", features = ["derive", "rc"] } @@ -22,7 +22,7 @@ serde_json = "1" tracing = "0.1" [dev-dependencies] -bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } tokio = { version = "1", features = ["time", "net", "rt", "macros"]} tokio-util = { version = "0.7.15", features = ["compat"] } tracing-subscriber = "0.3" diff --git a/trusted-headers-gen/Cargo.toml b/trusted-headers-gen/Cargo.toml index ca494c6..4e2bb01 100644 --- a/trusted-headers-gen/Cargo.toml +++ b/trusted-headers-gen/Cargo.toml @@ -9,9 +9,9 @@ repository = "https://github.com/evanlinjin/experiments" [dependencies] anyhow = "1" -bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } corepc-client = { version = "0.13", features = ["client-sync"] } serde_json = "1" [dev-dependencies] -bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", branch = "master" } +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } From 55b9b21c81399578348a837df1fabb5092e2c6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 2 Sep 2026 04:17:17 +0000 Subject: [PATCH 7/9] docs(bdk_electrum_streaming): Be exact about what a rebuilt status matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `from_wallet_txs` reproduces a server's status exactly for confirmed history, which is ordered by facts about the chain every server agrees on. Unconfirmed history is weaker: the protocol specifies an ordering, but a server is free to hash its mempool entries in whatever order they come out of its own index, so a script with several unconfirmed transactions may still hash to something it does not recognise — costing a refetch that would have happened anyway. Claiming it computes statuses "the same way Electrum does" oversold that. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013Ga32WdTURWnpMDxAckM2K --- bdk_electrum_streaming/src/cache.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bdk_electrum_streaming/src/cache.rs b/bdk_electrum_streaming/src/cache.rs index ee2722f..ea9c2c6 100644 --- a/bdk_electrum_streaming/src/cache.rs +++ b/bdk_electrum_streaming/src/cache.rs @@ -127,7 +127,14 @@ impl Cache { /// /// A caller cannot rebuild this from wallet data alone: a status is a hash Electrum computes /// over the history it stands for and no wallet stores, so [`Cache::from_wallet_txs`] computes -/// it the same way Electrum does. +/// it the way the protocol specifies instead. +/// +/// That reproduces a server's status exactly for confirmed history, which is ordered by height +/// and block position — both facts about the chain that every server agrees on. Unconfirmed +/// history is weaker: the protocol gives an ordering, but a server is free to hash its mempool +/// entries in whatever order they come out of its own index, so a script with more than one +/// unconfirmed transaction may still hash to something the server does not recognise. Costing +/// only a refetch of that script's history, which is what would have happened anyway. /// /// Fields are private: a status is a hash of the history it stands for, and letting the two be /// set independently would reintroduce the desync the type exists to prevent. From 9f44a92ae4f2c2288f560f16061f11584c9e48a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Sun, 6 Sep 2026 11:07:13 +0000 Subject: [PATCH 8/9] fix(bdk_electrum_streaming)!: Require a trusted set to include genesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing caught a trusted set built for the wrong network. Handed another chain's real headers, a network with the difficulty rules off and a high target limit accepts them in full: they link to each other, they hash below their claimed targets, and the retarget check that would notice never runs. The set is silently adopted as the anchor for everything above it. Genesis is the one block whose hash `params` fixes, so it is the only thing a set can be held against. Require it in any non-empty set, which makes the set say which chain it came from and turns the existing height-0 agreement check into one that actually fires. An empty set still means "sync from genesis" — it has nothing to mismatch. A generated headers file had no genesis entry, so the rule would have rejected the one file anyone passes. `trusted-headers-gen` now writes its network's genesis into every section it touches, derived from the network rather than fetched: it is a constant, and the node has already been held to that network by then. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HTtvv9UuTfnZXuMr2kL9bq --- bdk_electrum_streaming/src/header_chain.rs | 63 ++++++++++++++++++---- bdk_electrum_streaming/tests/env.rs | 5 +- bdk_electrum_streaming/tests/state.rs | 6 ++- trusted-headers-gen/src/main.rs | 28 ++++++++-- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/bdk_electrum_streaming/src/header_chain.rs b/bdk_electrum_streaming/src/header_chain.rs index 8bc4843..4ae6730 100644 --- a/bdk_electrum_streaming/src/header_chain.rs +++ b/bdk_electrum_streaming/src/header_chain.rs @@ -25,6 +25,9 @@ use bdk_core::{ /// block is included in the [`CheckPoint`] handed out by [`tip`](Self::tip), so it can always be /// connected to a `LocalChain`. Between the trusted blocks and the sync start there are gaps. /// +/// A non-empty set of trusted headers has to include genesis, so that it declares which network +/// it belongs to. See [`new`](Self::new). +/// /// # Difficulty /// /// (3) is what makes (4) mean anything: without it a server could claim a trivial difficulty and @@ -53,7 +56,9 @@ impl HeaderChain { /// the sync start to be verified later, when a transaction turns out to be confirmed down /// there; they can be at any height. /// - /// Genesis is added automatically; an entry at height `0` must agree with `params`. + /// A non-empty set must include genesis, and it must agree with `params` — that entry is + /// what says which network the set was built for, and without it a set from another chain is + /// accepted in full. An empty set means "sync from genesis". pub fn new( params: impl Into, trusted: impl IntoIterator, @@ -61,14 +66,27 @@ impl HeaderChain { let params = params.into(); let genesis = genesis_block(¶ms).header; let mut trusted = trusted.into_iter().collect::>(); - if let Some(header) = trusted.insert(0, genesis) { - ensure!( + // A non-empty set must name the chain it came from by including genesis. Nothing below + // can catch a set built for another network: on a network with the difficulty rules off + // and a high target limit, another chain's real headers link, hash and verify perfectly. + // Genesis is the one block whose hash is fixed by `params`, so it is the only thing the + // set can be held against. An empty set has nothing to mismatch. + match trusted.insert(0, genesis) { + Some(header) => ensure!( header.block_hash() == genesis.block_hash(), "trusted block at height 0 is {}, but {} has genesis {}", header.block_hash(), params.network, genesis.block_hash(), - ); + ), + // `trusted` now holds the genesis just inserted, so a length of one means it came in + // empty. + None => ensure!( + trusted.len() == 1, + "a trusted set must include the genesis block at height 0, so that it says which \ + network it is for; this one starts at height {}", + trusted.keys().nth(1).expect("more than one entry"), + ), } let anchor = *trusted .keys() @@ -405,10 +423,15 @@ mod test { forked[from as usize + 1..].to_vec() } + /// A [`HeaderChain`] trusting `trusted_heights` of `headers`, plus genesis, which every + /// non-empty trusted set has to carry. fn chain(headers: &[Header], trusted_heights: &[u32]) -> HeaderChain { HeaderChain::new( params(), - trusted_heights.iter().map(|&h| (h, headers[h as usize])), + trusted_heights + .iter() + .chain(&[0]) + .map(|&h| (h, headers[h as usize])), ) .unwrap() } @@ -486,7 +509,11 @@ mod test { // chain it replaces without tripping the difficulty rule. let params = Params::REGTEST; let headers = mine(¶ms, 10); - let mut c = HeaderChain::new(params.clone(), [(3, headers[3]), (8, headers[8])]).unwrap(); + let mut c = HeaderChain::new( + params.clone(), + [(0, headers[0]), (3, headers[3]), (8, headers[8])], + ) + .unwrap(); c.apply(9, headers[9..].to_vec()).unwrap(); c.apply(4, headers[4..9].to_vec()).unwrap(); assert_eq!(c.base_height(), 4); @@ -582,11 +609,29 @@ mod test { assert_eq!(c.tip_height(), Some(10)); } + /// A set built for another network is not caught by anything else: here the headers are + /// real, so they link and hash correctly, and the difficulty rules are off on this network. + /// Genesis is the only block whose hash `params` fixes, so requiring it is what makes the + /// mismatch visible. + #[test] + fn rejects_a_trusted_set_without_genesis() { + let headers = mine(¶ms(), 6); + let err = HeaderChain::new(params(), [(5, headers[5])]) + .unwrap_err() + .to_string(); + assert!(err.contains("must include the genesis block"), "{err}"); + assert!(err.contains("starts at height 5"), "{err}"); + + // Empty is still fine: it means "sync from genesis", and has nothing to mismatch. + let c = HeaderChain::new(params(), []).unwrap(); + assert_eq!(c.base_height(), 1); + } + #[test] fn rejects_a_trusted_anchor_off_the_retarget_boundary() { let params = retarget_params(); let headers = mine(¶ms, 12); - let err = HeaderChain::new(params, [(11, headers[11])]) + let err = HeaderChain::new(params, [(0, headers[0]), (11, headers[11])]) .unwrap_err() .to_string(); assert!(err.contains("difficulty-adjustment boundary"), "{err}"); @@ -601,7 +646,7 @@ mod test { "difficulty must actually move for this test to mean anything" ); - let mut c = HeaderChain::new(params.clone(), [(10, headers[10])]).unwrap(); + let mut c = HeaderChain::new(params.clone(), [(0, headers[0]), (10, headers[10])]).unwrap(); assert_eq!(c.base_height(), 11); c.apply(11, headers[11..].to_vec()).unwrap(); assert_eq!(c.tip_height(), Some(25)); @@ -609,7 +654,7 @@ mod test { // The retarget at 20 is recomputed from the trusted header at 10 — no gap on faith. let mut faked = headers.clone(); faked[20].bits = headers[19].bits; - let mut c = HeaderChain::new(params, [(10, headers[10])]).unwrap(); + let mut c = HeaderChain::new(params, [(0, headers[0]), (10, headers[10])]).unwrap(); let err = c.apply(11, faked[11..].to_vec()).unwrap_err().to_string(); assert!(err.contains("height 20"), "{err}"); assert!(err.contains("consensus requires"), "{err}"); diff --git a/bdk_electrum_streaming/tests/env.rs b/bdk_electrum_streaming/tests/env.rs index 223cac3..0f11b73 100644 --- a/bdk_electrum_streaming/tests/env.rs +++ b/bdk_electrum_streaming/tests/env.rs @@ -249,7 +249,10 @@ async fn backfills_history_below_the_sync_start() -> anyhow::Result<()> { ReqCoord::default(), Cache::default(), spk_tracker, - HeaderChain::new(Network::Regtest, [(trusted_height, trusted_header)])?, + HeaderChain::new( + Network::Regtest, + [(0, genesis_header()), (trusted_height, trusted_header)], + )?, ); assert_eq!( state.chain().base_height(), diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index 8b77505..8705026 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -296,7 +296,11 @@ fn new_state_trusting( ) -> BlockingState { let mut spk_tracker = DerivedSpkTracker::new(0); spk_tracker.insert_descriptor("external", descriptor, 0); - let chain = HeaderChain::new(Network::Regtest, trusted).expect("must build header chain"); + // Genesis goes in here rather than at every call site: a non-empty trusted set has to carry + // it, and an empty one is unchanged by it. + let genesis = constants::genesis_block(Network::Regtest).header; + let chain = HeaderChain::new(Network::Regtest, trusted.into_iter().chain([(0, genesis)])) + .expect("must build header chain"); BlockingState::new(ReqCoord::default(), cache, spk_tracker, chain) } diff --git a/trusted-headers-gen/src/main.rs b/trusted-headers-gen/src/main.rs index 7b6a57d..f495cfc 100644 --- a/trusted-headers-gen/src/main.rs +++ b/trusted-headers-gen/src/main.rs @@ -35,7 +35,13 @@ use std::{ str::FromStr, }; -use bdk_core::bitcoin::{block::Header, consensus::encode::deserialize_hex, BlockHash, Network}; +use bdk_core::bitcoin::{ + block::Header, + consensus::encode::{deserialize_hex, serialize_hex}, + constants::genesis_block, + params::Params, + BlockHash, Network, +}; use corepc_client::client_sync::{v17::Client, Auth}; /// Per-network state: trusted heights and the raw header hex at each. Rebuilt from the existing @@ -148,10 +154,9 @@ fn main() -> anyhow::Result<()> { let (height, hash, header_hex) = fetch_trusted_header(&client, args.network, args.height)?; let mut state = parse_existing(&args.out); - state - .entry(args.network) - .or_default() - .insert(height, header_hex); + let section = state.entry(args.network).or_default(); + section.insert(height, header_hex); + section.insert(0, genesis_hex(args.network)); fs::write(&args.out, render_file(&state))?; let _ = std::process::Command::new("rustfmt") .arg(&args.out) @@ -165,6 +170,19 @@ fn main() -> anyhow::Result<()> { Ok(()) } +/// This network's genesis header, hex-encoded. +/// +/// Every section carries one, so the file says which chain each of its headers came from. +/// `HeaderChain::new` refuses a trusted set that has no genesis, and checks the one it has +/// against the params it was handed — without that, a set built for one network can be handed to +/// another's params and pass every later check, because the headers themselves are real. +/// +/// Derived rather than fetched: it is a constant of the network, and the node has already been +/// held to that network by the time we get here. +fn genesis_hex(network: Network) -> String { + serialize_hex(&genesis_block(Params::from(network)).header) +} + /// Blocks between difficulty adjustments. const RETARGET_INTERVAL: u32 = 2016; From 2a1ecde506c7d7ebc8fd9f2366a5f98f26346a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Sun, 6 Sep 2026 11:07:17 +0000 Subject: [PATCH 9/9] feat(bdk_electrum_streaming): Re-export `anyhow` `anyhow::Error` appears throughout this crate's public signatures, but the crate was not re-exported, so a caller wanting to inspect an error had to add their own `anyhow` dependency and keep its version in step with ours. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HTtvv9UuTfnZXuMr2kL9bq --- bdk_electrum_streaming/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bdk_electrum_streaming/src/lib.rs b/bdk_electrum_streaming/src/lib.rs index 2918dca..0e06a09 100644 --- a/bdk_electrum_streaming/src/lib.rs +++ b/bdk_electrum_streaming/src/lib.rs @@ -6,6 +6,9 @@ use bdk_core::{ bitcoin::{block::Header, Txid}, spk_client::FullScanResponse, }; +/// Re-export, so callers can inspect the errors this crate returns without matching its +/// `anyhow` version themselves. +pub use anyhow; /// Re-export. pub use electrum_streaming_client;