diff --git a/Cargo.lock b/Cargo.lock index 7d86ac4..e1a5c0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -8,17 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.12" @@ -60,21 +49,20 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.1" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] -name = "base64ct" -version = "1.8.3" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bdk_chain" -version = "0.23.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c290eff038799a8ac0c5a82b6160a9ca456baa299a6f22b262c771342d2846c0" +version = "0.23.2" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_core", "bitcoin", @@ -83,9 +71,8 @@ dependencies = [ [[package]] name = "bdk_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3028782f6bf14a6df987244333d34e6b272b5a40a53e4879ec2dfd82275a3a" +version = "0.6.2" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bitcoin", "hashbrown", @@ -115,10 +102,10 @@ dependencies = [ [[package]] name = "bdk_testenv" version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543dc273dab3b9ec329772bcb15741a948cc3510deae2a30af3a116f03505fee" +source = "git+https://github.com/bitcoindevkit/bdk.git?rev=456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb#456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" dependencies = [ "bdk_chain", + "bitcoin", "electrsd", ] @@ -135,6 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0ce8bd5baaa0d303a19915a6d93afed161f528654e42da2a7a97d05c59499a" dependencies = [ "base58ck", + "base64 0.21.7", "bech32", "bitcoin-io", "bitcoin-units", @@ -192,42 +180,19 @@ dependencies = [ "serde", ] -[[package]] -name = "bitcoincore-rpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" -dependencies = [ - "bitcoincore-rpc-json", - "jsonrpc", - "log", - "serde", - "serde_json", -] - -[[package]] -name = "bitcoincore-rpc-json" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" -dependencies = [ - "bitcoin", - "serde", - "serde_json", -] - [[package]] name = "bitcoind" -version = "0.36.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce6620b7c942dbe28cc49c21d95e792feb9ffd95a093205e7875ccfa69c2925" +checksum = "deae3e9d99e37df8e82023e9f253cabe4b26f8a24a3e4e415f97a546f5280b66" dependencies = [ "anyhow", "bitcoin_hashes", - "bitcoincore-rpc", + "bitreq", + "corepc-client", "flate2", "log", - "minreq", + "serde_json", "tar", "tempfile", "which", @@ -247,12 +212,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "bitreq" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "b65c2e1ab98050c93cc9ada070d5070e014329c65196d03f6f8f1f75c2666f37" dependencies = [ - "generic-array", + "rustls", + "rustls-webpki", + "serde", + "serde_json", + "webpki-roots", ] [[package]] @@ -294,8 +263,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -306,28 +273,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "cipher" -version = "0.4.4" +name = "corepc-client" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "168e5075162cb8c253e8eea1575391c2ef8adde050bf9f3ef29d1cb109364388" dependencies = [ - "crypto-common", - "inout", + "bitcoin", + "corepc-types", + "jsonrpc", + "log", + "serde", + "serde_json", ] [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "corepc-types" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "1583872320eb2ac629c36753023fd072f1ca1b3b74b20cc62bab055b54278789" dependencies = [ - "libc", + "bitcoin", + "serde", + "serde_json", ] [[package]] @@ -345,60 +312,27 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "either" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" - [[package]] name = "electrsd" -version = "0.28.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c3c57645202a05a47206ed81cc179cf32bf4c3ca5a9e60c06c49d0222d5844" +checksum = "50d22a2f7bac981d425536cbeca21bc65b9d11a375c6e9ac853ccf61a1eb6119" dependencies = [ "bitcoin_hashes", "bitcoind", + "bitreq", + "corepc-client", "electrum-client", "log", - "minreq", "nix", - "which", "zip", ] [[package]] name = "electrum-client" -version = "0.20.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7b1f8783238bb18e6e137875b0a66f3dffe6c7ea84066e05d033cf180b150f" +checksum = "a5059f13888a90486e7268bbce59b175f5f76b1c55e5b9c568ceaa42d2b8507c" dependencies = [ "bitcoin", "log", @@ -556,16 +490,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -577,17 +501,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -622,57 +535,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - [[package]] name = "jsonrpc" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +checksum = "f106cca655869522988b976127096f0aa36e0f1a8ff1f1f425a5c85b61e59391" dependencies = [ - "base64", - "minreq", + "base64 0.22.1", + "bitreq", "serde", "serde_json", ] @@ -689,12 +565,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -724,12 +594,13 @@ dependencies = [ [[package]] name = "miniscript" -version = "12.3.7" +version = "13.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8343cc1ef1408bd9bdbf69f7aef47017dfab7e6349ec26fddf62e0e9fb5a4cf" +checksum = "cd35e2c377504e50159561884b03610711db6c2fec6c89f2b98d94016684726b" dependencies = [ "bech32", "bitcoin", + "hex-conservative 1.2.0", ] [[package]] @@ -742,19 +613,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "minreq" -version = "2.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05015102dad0f7d61691ca347e9d9d9006685a64aefb3d79eecf62665de2153d" -dependencies = [ - "rustls", - "rustls-webpki", - "serde", - "serde_json", - "webpki-roots", -] - [[package]] name = "mio" version = "1.2.2" @@ -789,41 +647,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -842,12 +671,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -875,12 +698,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "rand" version = "0.8.7" @@ -908,7 +725,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.17", + "getrandom", ] [[package]] @@ -919,25 +736,12 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.17", + "getrandom", "libc", "untrusted", "windows-sys 0.52.0", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -947,39 +751,41 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] -name = "rustls-webpki" -version = "0.101.7" +name = "rustls-pki-types" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ - "ring", - "untrusted", + "zeroize", ] [[package]] -name = "sct" -version = "0.7.1" +name = "rustls-webpki" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1047,28 +853,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -1158,9 +942,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -1173,25 +956,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "time" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - [[package]] name = "tokio" version = "1.53.1" @@ -1290,10 +1054,15 @@ dependencies = [ ] [[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +name = "trusted-headers-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "bdk_core", + "bdk_testenv", + "corepc-client", + "serde_json", +] [[package]] name = "unicode-ident" @@ -1327,20 +1096,20 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "webpki-roots" -version = "0.25.4" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "which" -version = "4.4.2" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", + "libc", ] [[package]] @@ -1358,15 +1127,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1447,7 +1207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1470,24 +1230,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zip" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes", "byteorder", "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd", ] [[package]] @@ -1495,32 +1254,3 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index 3649617..6eeee81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] resolver = "2" -members = ["bdk_electrum_streaming"] +members = ["bdk_electrum_streaming", "trusted-headers-gen"] diff --git a/bdk_electrum_streaming/Cargo.toml b/bdk_electrum_streaming/Cargo.toml index 45021b4..1b61f85 100644 --- a/bdk_electrum_streaming/Cargo.toml +++ b/bdk_electrum_streaming/Cargo.toml @@ -4,7 +4,7 @@ version = "0.7.0" description = "An async/blocking Electrum client for BDK, built as an explicit streaming state machine." license = "MIT OR Apache-2.0" edition = "2021" -rust-version = "1.70" +rust-version = "1.85" repository = "https://github.com/evanlinjin/experiments" documentation = "https://docs.rs/bdk_electrum_streaming" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" futures = "0.3" futures-timer = "3" anyhow = "1" -bdk_core = { version = "0.6", features = ["serde"] } -miniscript = { version = "12.0.0" } +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb", features = ["serde"] } +bdk_chain = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } +miniscript = { version = "13.0.0" } electrum_streaming_client = { version = "0.4" } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" tracing = "0.1" [dev-dependencies] -bdk_testenv = "0.13.0" -bdk_chain = "0.23.0" +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } tokio = { version = "1", features = ["time", "net", "rt", "macros"]} tokio-util = { version = "0.7.15", features = ["compat"] } tracing-subscriber = "0.3" diff --git a/bdk_electrum_streaming/README.md b/bdk_electrum_streaming/README.md index 296d96a..b7b4453 100644 --- a/bdk_electrum_streaming/README.md +++ b/bdk_electrum_streaming/README.md @@ -6,3 +6,17 @@ state machine on top of [`electrum_streaming_client`](https://docs.rs/electrum_s Tracks a set of descriptors, subscribes to their script hashes, and streams `FullScanResponse` updates as history, transactions and anchors resolve, instead of blocking until an entire scan completes. Handles reorgs by refetching whichever anchors they affect. + +The server is not trusted for chain data. You hand [`HeaderChain`] a set of trusted block headers; +everything above the highest one is downloaded and checked for linkage, proof-of-work, and the +difficulty consensus requires before it becomes part of the chain. A reorg is only accepted if it +brings more work than the blocks it replaces, and never if it would drop a trusted block. +Transactions are merkle-proved against those headers, and the proof travels with the anchor +([`ProvenAnchor`]). + +The highest trusted block must sit on a difficulty-adjustment boundary (`height % 2016 == 0`) on +networks where difficulty moves, so every retarget above it can be recomputed rather than taken on +faith. + +A transaction confirmed below the sync start triggers a backfill: headers are fetched from just +above the highest trusted block below it, up to where the chain already begins. diff --git a/bdk_electrum_streaming/src/anchor.rs b/bdk_electrum_streaming/src/anchor.rs new file mode 100644 index 0000000..adafa18 --- /dev/null +++ b/bdk_electrum_streaming/src/anchor.rs @@ -0,0 +1,33 @@ +use bdk_chain::Anchor; +use bdk_core::BlockId; +use electrum_streaming_client::DoubleSHA; + +/// Anchors a transaction to a block, recording the merkle proof that put it there. +/// +/// A [`ProvenAnchor`] only exists if [`merkle`](Self::merkle) was checked against the `merkle_root` +/// of the block's header, and that header is part of the verified +/// [`HeaderChain`](crate::HeaderChain). +/// +/// There is no block time here: the header the proof was checked against travels with every +/// [`Update`](crate::Update) as part of `chain_update`, so times can be read from there. +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct ProvenAnchor { + /// The block the transaction is proven to be in. + pub block_id: BlockId, + /// Position of the transaction within the block's merkle tree. + pub pos: usize, + /// Merkle branch connecting the transaction to the block's merkle root. + pub merkle: Vec, +} + +impl Anchor for ProvenAnchor { + fn anchor_block(&self) -> BlockId { + self.block_id + } + + fn confirmation_height_upper_bound(&self) -> u32 { + self.block_id.height + } +} diff --git a/bdk_electrum_streaming/src/cache.rs b/bdk_electrum_streaming/src/cache.rs index a8844fb..ea9c2c6 100644 --- a/bdk_electrum_streaming/src/cache.rs +++ b/bdk_electrum_streaming/src/cache.rs @@ -3,42 +3,21 @@ use std::{ sync::Arc, }; -use bdk_core::{ - bitcoin::{self, block::Header, BlockHash, Transaction, Txid}, - ConfirmationBlockTime, -}; -use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; +use bdk_core::bitcoin::{BlockHash, ScriptBuf, Transaction, Txid}; +use electrum_streaming_client::{response, ElectrumScriptHash, ElectrumScriptStatus}; + +use crate::ProvenAnchor; -/// Everything learned from the server, kept so a reconnect need not ask again. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +/// Everything learned from the server, so a job knows what it need not ask for again. +/// +/// Not persisted: every part of it is already in the caller's wallet — rebuild all of it with +/// [`Cache::from_wallet_txs`] — and a stored copy would only give the two something to disagree +/// about. +#[derive(Debug, Clone, Default)] pub struct Cache { /// The server's per-script histories. pub subscriptions: Subscriptions, - /// What we already hold, so a job knows what it need not ask for. - /// - /// Not persisted: every part of it is in the caller's wallet already, and a second copy - /// would only give the two something to disagree about. Seed it from wallet data instead. - #[serde(skip)] - pub tx_cache: TxCache, - - /// This can be removed once we can place `Header`s in `CheckPoint`s. - pub headers: HashMap, -} - -/// The transaction data a job consults before asking the server for anything. -/// -/// Separate from the rest of [`Cache`] because a caller can rebuild all of it from their own -/// wallet: the transactions are in their graph, the anchors with them, and which transactions -/// paid a script is what their spk index is for. So none of it is persisted alongside -/// [`Subscriptions`], which nothing can reconstruct. -/// -/// Starting empty is always correct, only expensive: a job asks the server for whatever it -/// cannot find here, so an empty one re-downloads every transaction and reproves every anchor. -/// It is not a mirror of the wallet, though — whatever a job fetches lands here too, so it -/// answers "do we already have this" whoever supplied it. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct TxCache { /// Every txid ever seen for each script hash. /// /// This is monotonically growing so that we can detect evictions. @@ -46,48 +25,116 @@ pub struct TxCache { pub txs: HashMap>, - /// Written as a sequence: a `(Txid, BlockHash)` key is not a string, so a map would be - /// unserializable in JSON and every other format that requires string keys. - #[serde(with = "persist::anchors_as_seq")] - pub anchors: HashMap<(Txid, BlockHash), ConfirmationBlockTime>, + pub anchors: HashMap<(Txid, BlockHash), ProvenAnchor>, +} + +/// A transaction's chain position, as a wallet already tracks it — enough to describe the +/// history a script had, without asking the server. +#[derive(Debug, Clone)] +pub enum TxConfirmationStatus { + /// Confirmed, with the proof a job would otherwise have to ask the server for again. + /// + /// `anchor.pos` plays no part in the status hash itself — only `txid` and `height` are + /// hashed — but a server orders a history by height *and then block position* (per the + /// [protocol]), so two transactions confirmed in the same block hash to a different status + /// if their order is wrong. + /// + /// [protocol]: https://electrum-protocol.readthedocs.io/en/latest/protocol-basics.html#status + Confirmed(ProvenAnchor), + Mempool { + confirmed_inputs: bool, + }, } impl Cache { - pub fn resolve_headers_query( - &mut self, - req: request::Headers, - resp: response::HeadersResp, - ) -> impl Iterator { - self.headers - .extend(resp.headers.iter().map(|&h| (h.block_hash(), h))); - (req.start_height..).zip(resp.headers) - } + /// Rebuild everything from what a wallet already knows: [`subscriptions`](Self::subscriptions), + /// `spk_txids`, `txs` and `anchors`. A reconnect need not download every script's history, + /// refetch its transactions, or reprove its anchors — only whatever actually changed. + /// + /// No `bdk_chain` dependency needed: `txs` is whatever a wallet's tx graph or index already + /// hands over as `(tx, status, relevant_scripts)` — one entry per transaction, not one per + /// script it pays, since that is how a wallet holds them. + pub fn from_wallet_txs( + txs: impl IntoIterator, + ) -> Self + where + Tx: Into>, + Spks: IntoIterator, + { + let mut cache = Cache::default(); + + // Sorted on later, by (height, block position) — carried alongside the response `Tx` + // since the wire type itself has nowhere to put it. + let mut by_spk_hash = HashMap::>::new(); + for (tx, status, relevant_scripts) in txs { + let tx = tx.into(); + let txid = tx.compute_txid(); + let (history_tx, sort_pos) = match status { + TxConfirmationStatus::Confirmed(anchor) => { + let history_tx = response::Tx::Confirmed(response::ConfirmedTx { + txid, + height: bdk_core::bitcoin::absolute::Height::from_consensus( + anchor.block_id.height, + ) + .expect("confirmed tx must have a valid height"), + }); + let pos = anchor.pos; + cache.anchors.insert((txid, anchor.block_id.hash), anchor); + (history_tx, pos) + } + TxConfirmationStatus::Mempool { confirmed_inputs } => ( + response::Tx::Mempool(response::MempoolTx { + txid, + fee: bdk_core::bitcoin::Amount::ZERO, + confirmed_inputs, + }), + 0, + ), + }; + cache.txs.insert(txid, tx); + for spk in relevant_scripts { + by_spk_hash + .entry(ElectrumScriptHash::new(&spk)) + .or_default() + .push((history_tx.clone(), sort_pos)); + } + } - pub fn resolve_history_query( - &mut self, - req: request::GetHistory, - resp: Vec, - ) -> Option { - let status_opt = ElectrumScriptStatus::from_history(&resp); - if let Some(status) = status_opt { - self.tx_cache + for (spk_hash, mut entries) in by_spk_hash { + // The order a server reports a history in, and the order its status hash is + // computed over: confirmed ascending by height then block position, then mempool + // ordered by confirmed-inputs before not, each tied by txid. + entries.sort_by_key(|(tx, pos)| match tx { + response::Tx::Confirmed(tx) => (0u8, tx.height.to_consensus_u32(), *pos, tx.txid), + response::Tx::Mempool(tx) if tx.confirmed_inputs => (1, 0, 0, tx.txid), + response::Tx::Mempool(tx) => (2, 0, 0, tx.txid), + }); + let history = entries.into_iter().map(|(tx, _)| tx).collect::>(); + cache .spk_txids - .entry(req.script_hash) + .entry(spk_hash) .or_default() - .extend(resp.iter().map(|tx| tx.txid())); - self.subscriptions.insert_spk(req.script_hash, status, resp); - } else { - self.subscriptions.remove_spk(req.script_hash); + .extend(history.iter().map(response::Tx::txid)); + if let Some(status) = ElectrumScriptStatus::from_history(&history) { + cache.subscriptions.insert_spk(spk_hash, status, history); + } } - status_opt + cache } } /// The last history the server reported for each script hash. /// -/// Unlike [`TxCache`], a caller cannot rebuild this from wallet data: a status is a hash Electrum -/// computes over the history it stands for and no wallet stores, and the server reports a history -/// as it stands now, never again mentioning a transaction it has dropped. +/// A caller cannot rebuild this from wallet data alone: a status is a hash Electrum computes +/// over the history it stands for and no wallet stores, so [`Cache::from_wallet_txs`] computes +/// it the way the protocol specifies instead. +/// +/// That reproduces a server's status exactly for confirmed history, which is ordered by height +/// and block position — both facts about the chain that every server agrees on. Unconfirmed +/// history is weaker: the protocol gives an ordering, but a server is free to hash its mempool +/// entries in whatever order they come out of its own index, so a script with more than one +/// unconfirmed transaction may still hash to something the server does not recognise. Costing +/// only a refetch of that script's history, which is what would have happened anyway. /// /// Fields are private: a status is a hash of the history it stands for, and letting the two be /// set independently would reintroduce the desync the type exists to prevent. @@ -180,136 +227,10 @@ impl Subscriptions { } } -/// Types and impls that exist only so [`Cache`] can be stored and loaded. -/// -/// Kept apart from the cache itself because [`HistoryTx`] mirrors [`response::Tx`] and the two -/// are easy to mistake for each other at a glance. -mod persist { - use super::*; - - /// A history entry in the shape we can write back out. - /// - /// [`response::Tx`] derives `Deserialize` only, so histories round-trip through this instead. - #[derive(serde::Serialize, serde::Deserialize)] - enum HistoryTx { - Mempool { - txid: Txid, - fee_sats: u64, - confirmed_inputs: bool, - }, - Confirmed { - txid: Txid, - height: u32, - }, - } - - impl From<&response::Tx> for HistoryTx { - fn from(tx: &response::Tx) -> Self { - match tx { - response::Tx::Mempool(tx) => Self::Mempool { - txid: tx.txid, - fee_sats: tx.fee.to_sat(), - confirmed_inputs: tx.confirmed_inputs, - }, - response::Tx::Confirmed(tx) => Self::Confirmed { - txid: tx.txid, - height: tx.height.to_consensus_u32(), - }, - } - } - } - - impl TryFrom for response::Tx { - type Error = bitcoin::absolute::ConversionError; - - fn try_from(tx: HistoryTx) -> Result { - Ok(match tx { - HistoryTx::Mempool { - txid, - fee_sats, - confirmed_inputs, - } => Self::Mempool(response::MempoolTx { - txid, - fee: bitcoin::Amount::from_sat(fee_sats), - confirmed_inputs, - }), - HistoryTx::Confirmed { txid, height } => Self::Confirmed(response::ConfirmedTx { - txid, - height: bitcoin::absolute::Height::from_consensus(height)?, - }), - }) - } - } - - /// Written as `spk_hash -> (status, history)`, which is what rebuilds both maps on the way - /// back in. - impl serde::Serialize for Subscriptions { - fn serialize(&self, serializer: S) -> Result { - serializer.collect_map(self.spk_hash_to_status.iter().map(|(&spk_hash, &status)| { - let history = self - .spk_status_to_history - .get(&status) - .map(|history| history.iter().map(HistoryTx::from).collect::>()) - .unwrap_or_default(); - (spk_hash, (status, history)) - })) - } - } - - impl<'de> serde::Deserialize<'de> for Subscriptions { - fn deserialize>(deserializer: D) -> Result { - use serde::de::Error; - let stored = - HashMap::)>::deserialize( - deserializer, - )?; - let mut spk_histories = Self::default(); - for (spk_hash, (status, history)) in stored { - let history = history - .into_iter() - .map(response::Tx::try_from) - .collect::, _>>() - .map_err(D::Error::custom)?; - spk_histories.insert_spk(spk_hash, status, history); - } - Ok(spk_histories) - } - } - - pub(super) mod anchors_as_seq { - use super::*; - use serde::{Deserialize, Deserializer, Serializer}; - - type Anchors = HashMap<(Txid, BlockHash), ConfirmationBlockTime>; - - pub fn serialize( - anchors: &Anchors, - serializer: S, - ) -> Result { - serializer.collect_seq( - anchors - .iter() - .map(|(&(txid, block_hash), anchor)| (txid, block_hash, anchor)), - ) - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result { - Ok( - Vec::<(Txid, BlockHash, ConfirmationBlockTime)>::deserialize(deserializer)? - .into_iter() - .map(|(txid, block_hash, anchor)| ((txid, block_hash), anchor)) - .collect(), - ) - } - } -} - #[cfg(test)] mod test { use super::*; - use bitcoin::hashes::Hash; + use bdk_core::bitcoin::{self, hashes::Hash}; fn txid(byte: u8) -> Txid { Txid::from_byte_array([byte; 32]) @@ -319,35 +240,25 @@ mod test { ElectrumScriptHash::from_byte_array([byte; 32]) } - /// `response::Tx` derives `Deserialize` only, so histories round-trip through `HistoryTx`. - /// Both of its variants have to survive the trip intact. - #[test] - fn spk_histories_round_trip() { - let history = vec![ - response::Tx::Confirmed(response::ConfirmedTx { - txid: txid(1), - height: bitcoin::absolute::Height::from_consensus(700_000).unwrap(), - }), - response::Tx::Mempool(response::MempoolTx { - txid: txid(2), - fee: bitcoin::Amount::from_sat(1234), - confirmed_inputs: false, - }), - ]; - let status = ElectrumScriptStatus::from_history(&history).expect("history is not empty"); - - let mut before = Subscriptions::default(); - before.insert_spk(spk_hash(9), status, history); - - let json = serde_json::to_string(&before).expect("must serialize"); - let after: Subscriptions = serde_json::from_str(&json).expect("must deserialize"); + /// A transaction whose txid varies with `unique`, so distinct calls yield distinct txids. + fn transaction(unique: u32) -> Transaction { + Transaction { + version: bitcoin::transaction::Version::ONE, + lock_time: bitcoin::absolute::LockTime::from_consensus(unique), + input: Vec::new(), + output: Vec::new(), + } + } - assert_eq!( - after.spk_history(status).map(<[_]>::len), - Some(2), - "the history must survive, and still answer to its status" - ); - assert_eq!(after.spk_status(spk_hash(9)), Some(status)); + fn anchor(height: u32, pos: usize) -> ProvenAnchor { + ProvenAnchor { + block_id: bdk_core::BlockId { + height, + hash: BlockHash::from_byte_array([height as u8; 32]), + }, + pos, + merkle: Vec::new(), + } } /// Two scripts paid by the same transaction, and nothing else, have identical histories — @@ -386,42 +297,157 @@ mod test { ); } - /// `anchors` is keyed by a tuple, which JSON cannot use as a map key. A caller who chooses - /// to persist a [`TxCache`] rather than rebuild it must still be able to. + /// A caller's wallet knows which spk each of its transactions paid, the transaction itself, + /// and whether it is confirmed (with its anchor) or still in the mempool — enough to + /// rebuild the transaction cache and compute the same status an Electrum server would, + /// without ever asking it. #[test] - fn tx_cache_round_trips_through_json() { - let anchor = (txid(1), bitcoin::BlockHash::from_byte_array([2; 32])); - let mut before = TxCache::default(); - before - .anchors - .insert(anchor, ConfirmationBlockTime::default()); + fn from_wallet_txs_rebuilds_a_status_the_server_would_recognise() { + let spk = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_hash = ElectrumScriptHash::new(&spk); + let confirmed_tx = transaction(1); + let mempool_tx = transaction(2); + let confirmed_txid = confirmed_tx.compute_txid(); + let mempool_txid = mempool_tx.compute_txid(); + let confirmed_anchor = anchor(100, 0); + + let cache = Cache::from_wallet_txs([ + ( + confirmed_tx, + TxConfirmationStatus::Confirmed(confirmed_anchor.clone()), + vec![spk.clone()], + ), + ( + mempool_tx, + TxConfirmationStatus::Mempool { + confirmed_inputs: true, + }, + vec![spk], + ), + ]); - let json = serde_json::to_string(&before).expect("must serialize"); - let after: TxCache = serde_json::from_str(&json).expect("must deserialize"); + assert_eq!( + cache.spk_txids.get(&spk_hash).map(BTreeSet::len), + Some(2), + "both txids must be recorded against the spk" + ); + assert_eq!( + cache.txs.keys().copied().collect::>(), + BTreeSet::from([confirmed_txid, mempool_txid]), + "the transactions themselves must be cached too" + ); + assert_eq!( + cache + .anchors + .get(&(confirmed_txid, confirmed_anchor.block_id.hash)), + Some(&confirmed_anchor), + "the confirmed transaction's anchor must be cached, needing no reproof" + ); - assert_eq!(after.anchors.get(&anchor), before.anchors.get(&anchor)); + let expected = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: confirmed_txid, + height: bitcoin::absolute::Height::from_consensus(100).unwrap(), + }), + response::Tx::Mempool(response::MempoolTx { + txid: mempool_txid, + fee: bitcoin::Amount::ZERO, + confirmed_inputs: true, + }), + ]) + .expect("history is not empty"); + assert_eq!( + cache.subscriptions.spk_status(spk_hash), + Some(expected), + "the rebuilt status must match what a server hashing the same history would report" + ); } - /// A `Cache` carries none of it, so persisting one cannot go stale against the wallet. + /// The status hash itself only ever mixes in `txid:height:` — never a block position — so + /// two transactions confirmed in the same block only get the right status if `pos` orders + /// them the way the block does. Get it backwards and the computed status is simply wrong. #[test] - fn cache_does_not_persist_the_tx_cache() { - let mut before = Cache::default(); - before.tx_cache.txs.insert( - txid(1), - Arc::new(bitcoin::Transaction { - version: bitcoin::transaction::Version::ONE, - lock_time: bitcoin::absolute::LockTime::ZERO, - input: Vec::new(), - output: Vec::new(), + fn same_block_transactions_are_ordered_by_block_position() { + let spk = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_hash = ElectrumScriptHash::new(&spk); + let (first_tx, second_tx) = (transaction(1), transaction(2)); + let (first, second) = (first_tx.compute_txid(), second_tx.compute_txid()); + let height = bitcoin::absolute::Height::from_consensus(100).unwrap(); + + let cache = Cache::from_wallet_txs([ + ( + first_tx, + TxConfirmationStatus::Confirmed(anchor(100, 0)), + vec![spk.clone()], + ), + ( + second_tx, + TxConfirmationStatus::Confirmed(anchor(100, 1)), + vec![spk], + ), + ]); + + let in_block_order = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: first, + height, }), + response::Tx::Confirmed(response::ConfirmedTx { + txid: second, + height, + }), + ]) + .expect("history is not empty"); + let reversed = ElectrumScriptStatus::from_history(&[ + response::Tx::Confirmed(response::ConfirmedTx { + txid: second, + height, + }), + response::Tx::Confirmed(response::ConfirmedTx { + txid: first, + height, + }), + ]) + .expect("history is not empty"); + assert_ne!( + in_block_order, reversed, + "the hash must actually be order-sensitive, or this test proves nothing" ); - let json = serde_json::to_string(&before).expect("must serialize"); - assert!( - !json.contains(&txid(1).to_string()), - "the wallet's own data must not be written here: {json}" + assert_eq!( + cache.subscriptions.spk_status(spk_hash), + Some(in_block_order), + "same-height entries must be ordered by their position in the block, not insertion \ + order, or the rebuilt status will not match what a server reports" ); - let after: Cache = serde_json::from_str(&json).expect("must deserialize"); - assert!(after.tx_cache.txs.is_empty()); + } + + /// A wallet holds each of its transactions once, not once per script it happens to pay — a + /// single entry naming every relevant script must still update every one of them. + #[test] + fn one_transaction_can_update_more_than_one_script() { + let spk_a = ScriptBuf::from_hex("0014000000000000000000000000000000000000000a").unwrap(); + let spk_b = ScriptBuf::from_hex("0014000000000000000000000000000000000000000b").unwrap(); + let tx = transaction(1); + let txid = tx.compute_txid(); + + let cache = Cache::from_wallet_txs([( + tx, + TxConfirmationStatus::Confirmed(anchor(100, 0)), + vec![spk_a.clone(), spk_b.clone()], + )]); + + for spk in [spk_a, spk_b] { + let spk_hash = ElectrumScriptHash::new(&spk); + assert_eq!( + cache.spk_txids.get(&spk_hash), + Some(&BTreeSet::from([txid])), + "the tx must be recorded against every script it pays" + ); + assert!( + cache.subscriptions.spk_status(spk_hash).is_some(), + "and every script must get a status to subscribe with" + ); + } } } diff --git a/bdk_electrum_streaming/src/confirmation_job.rs b/bdk_electrum_streaming/src/confirmation_job.rs index 0c5714f..c704e14 100644 --- a/bdk_electrum_streaming/src/confirmation_job.rs +++ b/bdk_electrum_streaming/src/confirmation_job.rs @@ -1,20 +1,24 @@ use std::collections::{BTreeMap, BTreeSet}; use bdk_core::{ - bitcoin::{block::Header, BlockHash, Txid}, + bitcoin::{block::Header, Txid}, BlockId, CheckPoint, }; use electrum_streaming_client::{request, ElectrumScriptStatus}; -use crate::{AnchorUpdate, Cache, ReqQueuer}; +use crate::{AnchorUpdate, Cache, HeaderChain, ReqQueuer}; /// How far along [`ConfirmationJob`] is. #[derive(Debug, Default, Clone)] pub enum ConfirmationStage { #[default] Init, - FetchBlocks { - to_fetch: BTreeSet, + /// Waiting on contiguous runs of headers, each `start -> end` inclusive. + /// + /// Runs, not scattered heights: [`HeaderChain`] verifies each header against the one below + /// it, so a header is only worth having as part of an unbroken run reaching the chain. + FetchHeaders { + runs: BTreeMap, }, FetchAnchors { to_fetch: BTreeSet<(u32, Txid)>, @@ -26,31 +30,14 @@ pub enum ConfirmationStage { Done, /// Nothing left to do until the target tip or the statuses move. /// - /// The update was taken, or the job was abandoned on inconsistent headers. Distinct from - /// [`Done`], which still owes one — a single stage for both would hand the same update over - /// twice, and hand one over for an abandoned job. + /// The update was taken, or the job was abandoned on a chain the server has left. Distinct + /// from [`Done`], which still owes one — a single stage for both would hand the same update + /// over twice, and hand one over for an abandoned job. /// /// [`Done`]: Self::Done Idle, } -impl ConfirmationStage { - pub fn fetch_anchors( - cache: &Cache, - spk_statuses: impl IntoIterator, - ) -> Self { - let to_fetch = cache - .subscriptions - .spk_histories(spk_statuses) - .filter_map(|tx| { - let conf_height = tx.confirmation_height()?.to_consensus_u32(); - Some((conf_height, tx.txid())) - }) - .collect(); - Self::FetchAnchors { to_fetch } - } -} - /// What one [`ConfirmationJob::poll`] achieved. /// /// The two `Update` variants are the parts of an [`Update`] this job owns; the rest come from @@ -58,12 +45,13 @@ impl ConfirmationStage { /// /// [`Update`]: crate::Update /// [`SpkJob`]: crate::SpkJob +#[derive(Debug)] pub enum ConfirmationProgress { - /// The local chain moved. - CheckPointUpdate { - cp: CheckPoint, - /// If there are any evictions, we need to check which spks need reanchoring - evicted: Vec, + /// The verified chain moved. + ChainUpdate { + cp: CheckPoint
, + /// Whether the run displaced blocks we already had. + reorged: bool, }, /// Every anchor the job set out to prove, resolved against one chain. AnchorUpdate(AnchorUpdate), @@ -75,7 +63,7 @@ pub enum ConfirmationProgress { Done, } -/// The single job that moves the local chain and anchors what the scripts found. +/// The single job that moves the verified chain and anchors what the scripts found. /// /// Runs once every [`SpkJob`] has its history — the heights those histories name are all it /// reads, so a script still downloading its own transactions has already told it every block it @@ -93,16 +81,28 @@ pub struct ConfirmationJob { target_header: Header, target_statuses: BTreeSet, - /// Always contains the target header; the notification carries it, so it is never fetched. + /// Headers as the server gave them, before [`HeaderChain`] has verified any of them. + /// + /// Kept apart from the chain deliberately: nothing here has been checked, and a header only + /// becomes part of the chain once its whole run passes. Always contains the target header; + /// the notification carries it, so it is never fetched. fetched_headers: BTreeMap, stage: ConfirmationStage, } impl ConfirmationJob { - /// An assumption of the max reorg depth. - const MAX_REORG_DEPTH: u32 = 21; + /// How far below the tip to re-download so a reorg is noticed. + /// + /// A reorg deeper than this is not walked back to; the run will not link to what we hold and + /// the connection errors out rather than quietly keeping a chain the server has left. The + /// verified chain is what stops a deep fork being adopted wrongly — it still has to out-work + /// what it replaces — but noticing one at all stops here. + /// + /// ponytail: fixed 21-block window, downloading every header and verifying work from genesis + /// removes the need for a window at all. + const REORG_WINDOW: u32 = 21; - /// Number of blocks before difficulty adjustment. + /// Most headers a server will hand over in one `blockchain.block.headers`. const MAX_BATCH_HEADERS_REQUEST: u32 = 2016; pub fn new(target_height: u32, target_header: Header) -> Self { @@ -196,156 +196,106 @@ impl ConfirmationJob { self.fetched_headers.extend(blocks); } - /// Polls the job as far as it will go. + /// Take one step towards a verified chain that covers everything worth anchoring. pub fn poll( &mut self, queuer: &mut ReqQueuer, cache: &Cache, - cp: &CheckPoint, + chain: &mut HeaderChain, ) -> anyhow::Result { match core::mem::take(&mut self.stage) { ConfirmationStage::Init => { - let to_fetch = self.missing_heights(cache, cp); - - // NOTE: This logic is not perfect and we may duplicate requests due to spk history - // changes between calls to `ConfirmationJob::poll`. Let's not fix it here as we will - // change this crate to download all headers and verify PoW later so there will be - // no need for this logic. - let mut start_height_opt = Option::::None; - let mut iter = to_fetch - .iter() - .copied() - .filter(|h| !self.fetched_headers.contains_key(h)) - .peekable(); - while let Some(h) = iter.next() { - if start_height_opt.is_none() { - start_height_opt = Some(h); - } - let start_height = start_height_opt.expect("must exist"); - if iter.peek().is_some_and(|&next_h| { - next_h <= h.saturating_add(1) - && next_h.saturating_sub(start_height) < Self::MAX_BATCH_HEADERS_REQUEST - }) { - continue; - } - queuer.enqueue(request::Headers { - start_height, - count: (h + 1).saturating_sub(start_height) as usize, - }); - start_height_opt = None; + self.ensure_target_is_reachable(chain)?; + let runs = self.required_runs(cache, chain); + for (&start, &end) in &runs { + self.queue_gaps(queuer, start, end); } - - self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + self.stage = ConfirmationStage::FetchHeaders { runs }; Ok(ConfirmationProgress::Continue) } - ConfirmationStage::FetchBlocks { to_fetch } => { - if !to_fetch - .iter() - .all(|h| self.fetched_headers.contains_key(h)) - { - self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + ConfirmationStage::FetchHeaders { runs } => { + let complete = runs.iter().all(|(&start, &end)| { + (start..=end).all(|h| self.fetched_headers.contains_key(&h)) + }); + if !complete { + // Ask again for whatever is still missing. A server may answer + // `blockchain.block.headers` with fewer headers than were asked for, and + // nothing else would re-queue them: `Init` is only re-entered when the + // target or the statuses move, which on a settled chain may be never. + // Requests still in flight are deduplicated, so this cannot pile up. + for (&start, &end) in &runs { + self.queue_gaps(queuer, start, end); + } + self.stage = ConfirmationStage::FetchHeaders { runs }; return Ok(ConfirmationProgress::Blocked); } - // Headers that disagree mean a reorg landed between fetches; wait to be told. - let mut iter = self - .fetched_headers - .iter() - .rev() - .take((Self::MAX_REORG_DEPTH + 1) as usize) - .peekable(); - while let Some((&height, header)) = iter.next() { - if let Some(&(&prev_height, prev_header)) = iter.peek() { - if prev_height + 1 == height - && prev_header.block_hash() != header.prev_blockhash - { - tracing::info!( - height, - prev_blockhash = header.prev_blockhash.to_string(), - actual_prev_blockhash = prev_header.block_hash().to_string(), - "Fetched headers are inconsistent. Reorg? Abandoning." - ); - self.reset_headers(); - self.stage = ConfirmationStage::Idle; - return Ok(ConfirmationProgress::Blocked); - } + // A run reaching the target must put the announced block at the announced + // height. Anything else is a chain we were never told about — the server has + // moved on, and moving is what makes it announce again, so abandon rather than + // retry. + if let Some(header) = self.fetched_headers.get(&self.target_height) { + if header.block_hash() != self.target_header.block_hash() { + tracing::info!( + height = self.target_height, + announced = self.target_header.block_hash().to_string(), + received = header.block_hash().to_string(), + "Headers describe a chain other than the one announced. Abandoning.", + ); + self.reset_headers(); + self.stage = ConfirmationStage::Idle; + return Ok(ConfirmationProgress::Blocked); } } - // Everything we hold is spliced in from the lowest header up. The target - // header is always one of them, so there is always a run to splice. - let start = self - .fetched_headers - .keys() - .next() - .copied() - .unwrap_or(self.target_height); - let mut extension = BTreeMap::::new(); - let mut base_opt = Option::::None; - for cp in cp.iter() { - if cp.height() < start { - base_opt = Some(cp); - break; - } - extension.insert(cp.height(), cp.hash()); + let was = chain.tip().map(|cp| (cp.height(), cp.hash())); + // Ascending, so a backfill run lands before the run that extends the tip — which + // is the order `HeaderChain::apply` needs, since backfill has to reach the base + // the other run may then move. + for (&start, &end) in &runs { + let headers = (start..=end) + .map(|h| self.fetched_headers[&h]) + .collect::>(); + chain.apply(start, headers)?; } - let new_blocks = self - .fetched_headers - .iter() - .map(|(&height, header)| (height, header.block_hash())); - extension.extend(new_blocks); - if extension.get(&0).is_some_and(|&genesis_hash| { - genesis_hash != cp.get(0).expect("genesis must exist").hash() - }) { - return Err(anyhow::anyhow!("server attempted to replace genesis")); - } - let extension = extension - .into_iter() - .map(|(height, hash)| BlockId { height, hash }); - let cp_update = match base_opt { - Some(base) => base.extend(extension).expect("must not error"), - None => CheckPoint::from_block_ids(extension).expect("must not error"), - }; - - let mut evicted_heights = Vec::::new(); - for cp in cp.iter() { - if cp_update - .get(cp.height()) - .is_some_and(|cp_update| cp_update == cp) - { - break; + let cp = match chain.tip() { + Some(cp) => cp.clone(), + None => { + self.stage = ConfirmationStage::Done; + return Ok(ConfirmationProgress::Done); } - evicted_heights.push(cp.height()); - } + }; + let reorged = + was.is_some_and(|(height, hash)| chain.block_hash(height) != Some(hash)); - self.stage = - ConfirmationStage::fetch_anchors(cache, self.target_statuses.iter().copied()); - Ok(ConfirmationProgress::CheckPointUpdate { - cp: cp_update, - evicted: evicted_heights, - }) + self.stage = self.anchor_stage(cache); + Ok(ConfirmationProgress::ChainUpdate { cp, reorged }) } ConfirmationStage::FetchAnchors { to_fetch } => { let mut resolved = AnchorUpdate::new(); let mut all_resolved = true; + let mut needs_backfill = false; for &(height, txid) in &to_fetch { - let header = match self.fetched_headers.get(&height) { + let header = match chain.header(height) { Some(header) => header, - // Not expected to fire: a changed history moves the status set, which - // sends the job back to `Init` to plan this height. Release goes back and - // fetches rather than assume which block this height holds. + // Below where the chain starts: it has to grow downwards before this + // height can be checked at all, which `Init` plans a run for. + None if height < chain.base_height() => { + needs_backfill = true; + continue; + } + // Above the verified tip. There is nothing to plan: a run only reaches + // the tip the server has announced, and this height is beyond it. The + // announcement that carries it is what re-polls this — going back to + // `Init` here would replan the same unreachable run forever. None => { - debug_assert!( - false, - "history named height {height}, which the header pass did not cover" - ); - self.stage = ConfirmationStage::Init; - return Ok(ConfirmationProgress::Continue); + all_resolved = false; + continue; } }; - match cache.tx_cache.anchors.get(&(txid, header.block_hash())) { - Some(&anchor) => { - resolved.insert((anchor, txid)); + match cache.anchors.get(&(txid, header.block_hash())) { + Some(anchor) => { + resolved.insert((anchor.clone(), txid)); } None => { all_resolved = false; @@ -353,6 +303,10 @@ impl ConfirmationJob { } } } + if needs_backfill { + self.stage = ConfirmationStage::Init; + return Ok(ConfirmationProgress::Continue); + } if !all_resolved { // The whole set is kept, not just what is left: each pass resolves all of it // afresh against the chain as it stands right then, so a reorg landing @@ -376,60 +330,100 @@ impl ConfirmationJob { } } - /// The heights we still need from the server. + /// Refuse a target too far below the verified tip to plan a run to. /// - /// Heights whose header is already reachable from `cp` and `cache` are absorbed into - /// `fetched_headers` on the way through, so what comes back is only the gap. - fn missing_heights(&mut self, cache: &Cache, cp: &CheckPoint) -> BTreeSet { - let mut to_fetch = BTreeSet::::new(); + /// [`Self::required_runs`] only reaches back a [`REORG_WINDOW`](Self::REORG_WINDOW) below + /// the verified tip, so a target under that plans no run to it at all. Left alone the job + /// would find its remaining runs complete, apply nothing, and hand the caller back the + /// chain it already had as though the server had just confirmed it — then go on asking that + /// server for proofs at heights it does not have. A server that far behind cannot answer + /// for this wallet, whether it is mid-sync, was rolled back, or is one of several behind + /// one endpoint at different heights. + /// + /// Only checked once there *is* a verified tip. Before that there is no window to fall + /// below, and planning nothing is an ordinary waypoint rather than a fault: a target at or + /// under the block the chain starts at plans its backfill from the heights a history names, + /// which the histories have yet to arrive to name. + fn ensure_target_is_reachable(&self, chain: &HeaderChain) -> anyhow::Result<()> { + if let Some(tip) = chain.tip_height() { + anyhow::ensure!( + self.target_height >= tip.saturating_sub(Self::REORG_WINDOW), + "server's tip is {}, more than {} blocks below the verified tip {tip}", + self.target_height, + Self::REORG_WINDOW, + ); + } + Ok(()) + } - // Heights the chain itself has to be checked at. Settled first, because a height in - // here is one whose block may be about to be replaced — absorbing it from `cp` below - // would answer the question with the very block under suspicion. - if self.target_tip() != cp.block_id() { - let only_extends_tip = self - .target_height - .checked_sub(1) - .map(|height| { - let hash = self.target_header.prev_blockhash; - BlockId { height, hash } - }) - .is_some_and(|prev| cp.block_id() == prev); - if only_extends_tip { - to_fetch.extend(cp.height() + 1..=self.target_height); - } else { - // Assumes no reorg is deeper than `MAX_REORG_DEPTH`. - let old_tip = cp.height(); - let new_tip = self.target_height; - to_fetch.extend(old_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=old_tip); - to_fetch.extend(new_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=new_tip); + /// The heights carrying a transaction we have to anchor. + fn anchor_heights(&self, cache: &Cache) -> BTreeSet<(u32, Txid)> { + cache + .subscriptions + .spk_histories(self.target_statuses.iter().copied()) + .filter_map(|tx| Some((tx.confirmation_height()?.to_consensus_u32(), tx.txid()))) + .collect() + } + + fn anchor_stage(&self, cache: &Cache) -> ConfirmationStage { + ConfirmationStage::FetchAnchors { + to_fetch: self.anchor_heights(cache), + } + } + + /// The contiguous runs of headers the chain needs before every anchor can be checked. + /// + /// At most two: one up to the announced tip, and one backfilling history below where the + /// chain currently starts. Both are runs rather than the individual heights that want them, + /// because a header is only verifiable as part of a chain reaching a block we trust. + fn required_runs(&self, cache: &Cache, chain: &HeaderChain) -> BTreeMap { + let mut runs = BTreeMap::new(); + + let base = chain.base_height(); + let start = match chain.tip_height() { + // Re-download a window below the tip, so a reorg within it is seen at all. Never + // below the base: the run has to stay contiguous with what is already verified. + Some(tip) => base.max(tip.saturating_sub(Self::REORG_WINDOW)), + None => base, + }; + if start <= self.target_height { + runs.insert(start, self.target_height); + } + + // A transaction confirmed below where the chain starts cannot be checked against it, so + // the chain has to grow downwards to a block we already trust. + if let Some(&(lowest, _)) = self.anchor_heights(cache).iter().next() { + if lowest < base { + let from = chain.trusted_at_or_below(lowest).unwrap_or(0) + 1; + if from < base { + runs.insert(from, base - 1); + } } } - // Heights that carry a transaction to anchor. One the chain already places, and whose - // header we have, needs no request. - let anchor_heights = self - .target_statuses - .iter() - .filter_map(|&spk_status| { - let heights = cache - .subscriptions - .spk_history(spk_status)? - .iter() - .filter_map(|tx| Some(tx.confirmation_height()?.to_consensus_u32())); - Some(heights) - }) - .flatten() - .collect::>(); - for height in anchor_heights { - if !to_fetch.insert(height) { + runs + } + + /// Queue whatever part of `start..=end` we do not already hold, in server-sized batches. + fn queue_gaps(&self, queuer: &mut ReqQueuer, start: u32, end: u32) { + let mut height = start; + while height <= end { + if self.fetched_headers.contains_key(&height) { + height += 1; continue; } - if let Some(&header) = cp.get(height).and_then(|cp| cache.headers.get(&cp.hash())) { - self.fetched_headers.insert(height, header); + let mut count = 0; + while height + count <= end + && count < Self::MAX_BATCH_HEADERS_REQUEST + && !self.fetched_headers.contains_key(&(height + count)) + { + count += 1; } + queuer.enqueue(request::Headers { + start_height: height, + count: count as usize, + }); + height += count; } - - to_fetch } } diff --git a/bdk_electrum_streaming/src/header_chain.rs b/bdk_electrum_streaming/src/header_chain.rs new file mode 100644 index 0000000..4ae6730 --- /dev/null +++ b/bdk_electrum_streaming/src/header_chain.rs @@ -0,0 +1,662 @@ +use std::collections::BTreeMap; + +use anyhow::{ensure, Context}; +use bdk_core::{ + bitcoin::{ + block::Header, constants::genesis_block, params::Params, BlockHash, CompactTarget, Work, + }, + CheckPoint, +}; + +/// A chain of block headers, verified against a set of user-provided trusted headers. +/// +/// Every applied header must: +/// 1. match the trusted header at its height (if the user provided one), +/// 2. link to the block below it via `prev_blockhash`, +/// 3. claim the difficulty that consensus requires at its height, and +/// 4. hash below the target of that difficulty. +/// +/// Syncing starts one block above the highest trusted block, so (2) pins the very first header we +/// download to a block the user vouched for. Everything from there up is verified, contiguous, and +/// kept in memory. A reorg on top of that is only accepted if it brings more work than the blocks +/// it replaces. +/// +/// Genesis is trusted implicitly (it is derived from `params`, not downloaded) and every trusted +/// block is included in the [`CheckPoint`] handed out by [`tip`](Self::tip), so it can always be +/// connected to a `LocalChain`. Between the trusted blocks and the sync start there are gaps. +/// +/// A non-empty set of trusted headers has to include genesis, so that it declares which network +/// it belongs to. See [`new`](Self::new). +/// +/// # Difficulty +/// +/// (3) is what makes (4) mean anything: without it a server could claim a trivial difficulty and +/// mine a fake chain cheaply. Recomputing a retarget needs the header that opened the previous +/// difficulty period, so on networks where difficulty actually moves, the highest trusted block is +/// required to sit on a difficulty-adjustment boundary (`height % 2016 == 0`). That way every +/// retarget above it is recomputed from a header we already have, with no gap taken on faith. +/// +/// Trusted blocks *below* the sync start are exempt: a backfilled run is pinned by a trusted block +/// at the bottom and the verified chain at the top, so its difficulty needs no checking. +#[derive(Debug, Clone)] +pub struct HeaderChain { + params: Params, + /// Headers the user vouches for, plus genesis. Never downloaded, never replaced. + trusted: BTreeMap, + /// Lowest height of the contiguous verified segment. + base: u32, + cp: Option>, +} + +impl HeaderChain { + /// Construct a [`HeaderChain`] that trusts `trusted` (height to header). + /// + /// The highest entry decides where syncing starts, and must sit on a difficulty-adjustment + /// boundary on networks where difficulty moves. Lower entries are what allow history *below* + /// the sync start to be verified later, when a transaction turns out to be confirmed down + /// there; they can be at any height. + /// + /// A non-empty set must include genesis, and it must agree with `params` — that entry is + /// what says which network the set was built for, and without it a set from another chain is + /// accepted in full. An empty set means "sync from genesis". + pub fn new( + params: impl Into, + trusted: impl IntoIterator, + ) -> anyhow::Result { + let params = params.into(); + let genesis = genesis_block(¶ms).header; + let mut trusted = trusted.into_iter().collect::>(); + // A non-empty set must name the chain it came from by including genesis. Nothing below + // can catch a set built for another network: on a network with the difficulty rules off + // and a high target limit, another chain's real headers link, hash and verify perfectly. + // Genesis is the one block whose hash is fixed by `params`, so it is the only thing the + // set can be held against. An empty set has nothing to mismatch. + match trusted.insert(0, genesis) { + Some(header) => ensure!( + header.block_hash() == genesis.block_hash(), + "trusted block at height 0 is {}, but {} has genesis {}", + header.block_hash(), + params.network, + genesis.block_hash(), + ), + // `trusted` now holds the genesis just inserted, so a length of one means it came in + // empty. + None => ensure!( + trusted.len() == 1, + "a trusted set must include the genesis block at height 0, so that it says which \ + network it is for; this one starts at height {}", + trusted.keys().nth(1).expect("more than one entry"), + ), + } + let anchor = *trusted + .keys() + .next_back() + .expect("genesis was just inserted"); + let interval = params.difficulty_adjustment_interval() as u32; + if retargets(¶ms) { + ensure!( + anchor % interval == 0, + "the highest trusted block must sit on a difficulty-adjustment boundary (a \ + multiple of {interval}) so that every retarget above it can be recomputed; \ + height {anchor} is not one", + ); + } + Ok(Self { + params, + trusted, + base: anchor + 1, + cp: None, + }) + } + + fn interval(&self) -> u32 { + self.params.difficulty_adjustment_interval() as u32 + } + + /// The lowest height we need to download a header for. + /// + /// This starts one block above the highest trusted block and moves down as history is + /// backfilled. + pub fn base_height(&self) -> u32 { + self.base + } + + /// The highest trusted height at or below `height`. + pub fn trusted_at_or_below(&self, height: u32) -> Option { + self.trusted.range(..=height).next_back().map(|(&h, _)| h) + } + + /// The verified tip, if we have one. + /// + /// Every trusted block is in it, so it can be applied to a `LocalChain`. + pub fn tip(&self) -> Option<&CheckPoint
> { + self.cp.as_ref() + } + + /// Height of the verified tip, if we have one. + pub fn tip_height(&self) -> Option { + self.cp.as_ref().map(CheckPoint::height) + } + + /// The trusted or verified header at `height`, if we have it. + pub fn header(&self, height: u32) -> Option
{ + if let Some(&header) = self.trusted.get(&height) { + return Some(header); + } + self.cp.as_ref()?.get(height).map(|cp| cp.data()) + } + + /// The trusted or verified blockhash at `height`, if we have it. + pub fn block_hash(&self, height: u32) -> Option { + self.header(height).map(|h| h.block_hash()) + } + + /// Apply a contiguous, ascending run of `headers` beginning at `start`. + /// + /// The run may extend the tip, replace it (reorg), or sit below the current + /// [`base_height`](Self::base_height) to backfill history — in which case it must stop + /// exactly where the chain already begins, filling the gap and no more. + /// + /// A backfill is rebuilt rather than compared for work, because the blocks it adds sit + /// below everything already verified and displace nothing. A run that started below the + /// base and carried on past the tip would take that path while replacing verified blocks, + /// so it is refused: reorging the tip is the extend path's business, where the replacement + /// has to out-work what it replaces. + /// + /// The chain is left untouched if anything fails to verify. + pub fn apply(&mut self, start: u32, headers: Vec
) -> anyhow::Result<()> { + if headers.is_empty() { + return Ok(()); + } + ensure!(start > 0, "genesis is never applied"); + self.verify(start, &headers)?; + + let end = start + headers.len() as u32 - 1; + let old_tip_height = self.tip_height(); + let run = (start..).zip(headers); + // Trusted blocks below the run keep the checkpoint connectable to a `LocalChain`. + let below = self + .trusted + .range(..start) + .map(|(&height, &header)| (height, header)) + .collect::>(); + + let cp = match self.cp.clone() { + // Backfill: rebuild as the trusted blocks, the run, then whatever sat above the run. + Some(cp) if start < self.base => { + // Exactly, not merely far enough: a run that carried on past the base would + // replace verified blocks here, where nothing compares work. + ensure!( + end + 1 == self.base, + "a backfill must stop where the chain begins, at {}, but these headers \ + stop at {end}", + self.base - 1, + ); + let above = cp + .iter() + .take_while(|cp| cp.height() > end) + .collect::>(); + build( + below + .into_iter() + .chain(run) + .chain(above.into_iter().rev().map(|cp| (cp.height(), cp.data()))), + )? + } + // Extend the tip, evicting any block the run disagrees with. + Some(old) => { + ensure!( + start <= old.height() + 1, + "headers starting at {start} would leave a gap above tip {}", + old.height() + ); + let cp = run.fold(old.clone(), |cp, (height, header)| { + cp.insert(height, header) + }); + // Blocks below `start` are untouched, so comparing the chains from there is the + // same as comparing their totals. + let evicts_old_tip = cp + .get(old.height()) + .is_none_or(|cp| cp.hash() != old.hash()); + if evicts_old_tip { + ensure!( + work_from(&cp, start) > work_from(&old, start), + "the run reorgs our chain from height {start} without more work than the \ + blocks it replaces", + ); + } + cp + } + // Nothing yet: the trusted blocks plus the run become the chain. + None => build(below.into_iter().chain(run))?, + }; + + // An eviction must never take out a block we already trust. + if let Some(old_tip_height) = old_tip_height { + for (&height, header) in self.trusted.range(..=old_tip_height) { + let hash = header.block_hash(); + ensure!( + cp.get(height).is_some_and(|cp| cp.hash() == hash), + "the applied headers displace the trusted block at height {height} ({hash})" + ); + } + } + + self.base = self.base.min(start); + self.cp = Some(cp); + Ok(()) + } + + fn verify(&self, start: u32, headers: &[Header]) -> anyhow::Result<()> { + // Look up a header by height, preferring the run being verified over what we hold. + let at = |height: u32| -> Option
{ + height + .checked_sub(start) + .and_then(|i| headers.get(i as usize).copied()) + .or_else(|| self.header(height)) + }; + + for (i, header) in headers.iter().enumerate() { + let height = start + i as u32; + let hash = header.block_hash(); + + if let Some(trusted) = self.trusted.get(&height) { + ensure!( + hash == trusted.block_hash(), + "block {hash} at height {height} conflicts with trusted block {}", + trusted.block_hash(), + ); + } + if let Some(prev) = height.checked_sub(1).and_then(at) { + let prev_hash = prev.block_hash(); + ensure!( + header.prev_blockhash == prev_hash, + "block {hash} at height {height} does not link to {prev_hash} below it" + ); + } + if let Some(bits) = self.required_bits(height, at) { + ensure!( + header.bits == bits, + "block {hash} at height {height} claims difficulty {:#x}, consensus requires {:#x}", + header.bits.to_consensus(), + bits.to_consensus(), + ); + } + let target = header.target(); + ensure!( + target <= self.params.max_attainable_target, + "block {hash} at height {height} claims a target above the proof-of-work limit" + ); + header + .validate_pow(target) + .with_context(|| format!("block {hash} at height {height}"))?; + } + Ok(()) + } + + /// The difficulty consensus requires at `height`. + /// + /// Difficulty is fixed for a whole retarget period and recomputed at each boundary. Returns + /// `None` when the rule cannot be enforced: on networks that allow min-difficulty blocks, or + /// when we are missing a header the calculation needs — which, above the trusted anchor, + /// cannot happen, since the anchor sits on a boundary. + fn required_bits( + &self, + height: u32, + at: impl Fn(u32) -> Option
, + ) -> Option { + if self.params.allow_min_difficulty_blocks { + return None; + } + let prev = at(height.checked_sub(1)?)?; + let interval = self.interval(); + if height % interval != 0 { + return Some(prev.bits); + } + let boundary = at(height.checked_sub(interval)?)?; + Some(CompactTarget::from_header_difficulty_adjustment( + boundary, + prev, + &self.params, + )) + } +} + +/// Whether difficulty actually moves on this network. +fn retargets(params: &Params) -> bool { + !params.allow_min_difficulty_blocks && !params.no_pow_retargeting +} + +/// Total work of `cp` from `from` up to its tip. +fn work_from(cp: &CheckPoint
, from: u32) -> Work { + cp.range(from..) + .fold(Work::from_be_bytes([0; 32]), |sum, cp| { + sum + cp.data().work() + }) +} + +fn build(blocks: impl IntoIterator) -> anyhow::Result> { + CheckPoint::from_blocks(blocks).map_err(|_| anyhow::anyhow!("headers do not form a chain")) +} + +#[cfg(test)] +mod test { + use super::*; + use bdk_core::bitcoin::{block::Version, hashes::Hash, CompactTarget, TxMerkleNode}; + + /// Regtest, but with the difficulty rules switched on so they actually get exercised. + /// Retargeting stays off, so trusted blocks may sit anywhere. + fn params() -> Params { + let mut params = Params::REGTEST; + params.allow_min_difficulty_blocks = false; + params + } + + /// Regtest with real retargeting over a 10-block period, so retargets are cheap to mine. + fn retarget_params() -> Params { + let mut params = params(); + params.no_pow_retargeting = false; + params.pow_target_timespan = 10 * params.pow_target_spacing; + params + } + + /// Append `n` mined headers to `chain` (indexed by height), retargeting per `params`. + /// + /// `tag` distinguishes otherwise-identical forks. `bits` overrides the difficulty, which only + /// makes sense on a network that allows min-difficulty blocks. + fn extend( + params: &Params, + chain: &mut Vec
, + n: usize, + tag: u8, + bits: Option, + ) { + let interval = params.difficulty_adjustment_interval() as u32; + for _ in 0..n { + let height = chain.len() as u32; + let prev = chain[height as usize - 1]; + let bits = bits.unwrap_or(if height % interval == 0 { + CompactTarget::from_header_difficulty_adjustment( + chain[(height - interval) as usize], + prev, + params, + ) + } else { + prev.bits + }); + let mut merkle_root = [0u8; 32]; + merkle_root[0] = tag; + merkle_root[1] = height as u8; + let mut header = Header { + version: Version::ONE, + prev_blockhash: prev.block_hash(), + merkle_root: TxMerkleNode::from_byte_array(merkle_root), + time: prev.time + 600, + bits, + nonce: 0, + }; + // Grind for real proof-of-work. + while header.validate_pow(header.target()).is_err() { + header.nonce += 1; + } + chain.push(header); + } + } + + /// Genesis plus `n` mined headers. `chain[h]` is the header at height `h`. + fn mine(params: &Params, n: usize) -> Vec
{ + let mut chain = vec![genesis_block(params).header]; + extend(params, &mut chain, n, 0, None); + chain + } + + /// A fork of `chain` that branches above `from`, `n` blocks long. + fn fork( + params: &Params, + chain: &[Header], + from: u32, + n: usize, + bits: Option, + ) -> Vec
{ + let mut forked = chain[..=from as usize].to_vec(); + extend(params, &mut forked, n, 1, bits); + forked[from as usize + 1..].to_vec() + } + + /// A [`HeaderChain`] trusting `trusted_heights` of `headers`, plus genesis, which every + /// non-empty trusted set has to carry. + fn chain(headers: &[Header], trusted_heights: &[u32]) -> HeaderChain { + HeaderChain::new( + params(), + trusted_heights + .iter() + .chain(&[0]) + .map(|&h| (h, headers[h as usize])), + ) + .unwrap() + } + + #[test] + fn accepts_a_valid_chain() { + let headers = mine(¶ms(), 10); + let mut c = chain(&headers, &[5]); + assert_eq!(c.base_height(), 6, "sync starts above the trusted block"); + c.apply(6, headers[6..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(10)); + assert_eq!(c.header(7), Some(headers[7])); + assert_eq!( + c.tip().unwrap().iter().last().unwrap().height(), + 0, + "genesis is always the base" + ); + assert_eq!( + c.tip().unwrap().get(5).map(|cp| cp.hash()), + Some(headers[5].block_hash()), + "trusted blocks are in the checkpoint" + ); + } + + #[test] + fn rejects_a_run_that_does_not_link_to_the_trusted_block() { + let headers = mine(¶ms(), 10); + let forked = fork(¶ms(), &headers, 4, 6, None); + let mut c = chain(&headers, &[5]); + let err = c.apply(6, forked[1..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("does not link to"), "{err}"); + } + + #[test] + fn rejects_bad_pow() { + let mut headers = mine(¶ms(), 3); + let mut c = chain(&headers, &[]); + while headers[2].validate_pow(headers[2].target()).is_ok() { + headers[2].nonce = headers[2].nonce.wrapping_add(1); + } + let err = c.apply(1, headers[1..3].to_vec()).unwrap_err().to_string(); + assert!(err.contains("height 2"), "{err}"); + assert!(c.tip().is_none()); + } + + #[test] + fn rejects_broken_link() { + let mut headers = mine(¶ms(), 3); + let mut c = chain(&headers, &[]); + headers[3].prev_blockhash = BlockHash::all_zeros(); + assert!(c.apply(1, headers[1..].to_vec()).is_err()); + } + + #[test] + fn rejects_difficulty_change_within_a_period() { + let mut headers = mine(¶ms(), 2); + let mut c = chain(&headers, &[]); + headers[2].bits = CompactTarget::from_consensus(0x207ffffe); + let err = c.apply(1, headers[1..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("consensus requires"), "{err}"); + } + + #[test] + fn rejects_conflict_with_trusted_blockhash() { + let headers = mine(¶ms(), 4); + let forked = fork(¶ms(), &headers, 1, 3, None); + let mut c = chain(&headers, &[3]); + let err = c.apply(2, forked).unwrap_err().to_string(); + assert!(err.contains("conflicts with trusted"), "{err}"); + } + + #[test] + fn rejects_reorg_that_displaces_a_trusted_block() { + // Plain regtest: min-difficulty blocks are allowed, so a fork may be harder than the + // chain it replaces without tripping the difficulty rule. + let params = Params::REGTEST; + let headers = mine(¶ms, 10); + let mut c = HeaderChain::new( + params.clone(), + [(0, headers[0]), (3, headers[3]), (8, headers[8])], + ) + .unwrap(); + c.apply(9, headers[9..].to_vec()).unwrap(); + c.apply(4, headers[4..9].to_vec()).unwrap(); + assert_eq!(c.base_height(), 4); + + // Two blocks at 4x the difficulty out-work the seven they replace, and say nothing about + // height 8 — but they would drop it. + let harder = CompactTarget::from_consensus(0x201f_ffff); + let forked = fork(¶ms, &headers, 3, 2, Some(harder)); + let err = c.apply(4, forked).unwrap_err().to_string(); + assert!(err.contains("displace the trusted block"), "{err}"); + assert_eq!(c.tip_height(), Some(10), "chain is left untouched"); + assert_eq!(c.header(5), Some(headers[5])); + } + + #[test] + fn accepts_a_reorg_with_more_work() { + let headers = mine(¶ms(), 6); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + let forked = fork(¶ms(), &headers, 4, 3, None); + c.apply(5, forked.clone()).unwrap(); + assert_eq!(c.tip_height(), Some(7)); + assert_eq!(c.header(5), Some(forked[0])); + } + + #[test] + fn rejects_a_reorg_with_less_work() { + let headers = mine(¶ms(), 8); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + // Every block here carries the same work, so a shorter fork is a weaker chain. + let forked = fork(¶ms(), &headers, 4, 2, None); + let err = c.apply(5, forked).unwrap_err().to_string(); + assert!(err.contains("without more work"), "{err}"); + assert_eq!(c.tip_height(), Some(8), "chain is left untouched"); + assert_eq!(c.header(6), Some(headers[6])); + } + + #[test] + fn re_applying_the_same_headers_is_not_a_reorg() { + let headers = mine(¶ms(), 8); + let mut c = chain(&headers, &[4]); + c.apply(5, headers[5..].to_vec()).unwrap(); + // The reorg window re-downloads blocks we already have; equal work must still be fine. + c.apply(5, headers[5..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(8)); + } + + #[test] + fn backfills_below_the_base() { + let headers = mine(¶ms(), 12); + let mut c = chain(&headers, &[3, 8]); + c.apply(9, headers[9..].to_vec()).unwrap(); + assert_eq!(c.base_height(), 9); + assert_eq!(c.header(5), None); + assert_eq!(c.trusted_at_or_below(5), Some(3)); + + c.apply(4, headers[4..9].to_vec()).unwrap(); + assert_eq!(c.base_height(), 4); + assert_eq!(c.tip_height(), Some(12)); + assert_eq!(c.header(5), Some(headers[5])); + assert_eq!(c.tip().unwrap().iter().last().unwrap().height(), 0); + } + + #[test] + fn rejects_backfill_that_does_not_reach_the_base() { + let headers = mine(¶ms(), 12); + let mut c = chain(&headers, &[3, 8]); + c.apply(9, headers[9..].to_vec()).unwrap(); + assert!(c.apply(4, headers[4..7].to_vec()).is_err()); + } + + /// A run starting below the base takes the backfill path, which rebuilds the chain rather + /// than comparing work — so a run that also reaches past the tip could swap out verified + /// blocks for an equal-work fork, the very thing + /// [`rejects_a_reorg_with_less_work`](Self::rejects_a_reorg_with_less_work) forbids on the + /// extend path. A backfill has to fill the gap and stop. + #[test] + fn rejects_backfill_that_overshoots_the_base() { + let headers = mine(¶ms(), 10); + let mut c = chain(&headers, &[3]); + c.apply(4, headers[4..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(10)); + + // Real history up to the trusted block, then a fork of equal length — and so equal + // work — over everything above it. + let forked = fork(¶ms(), &headers, 3, 7, None); + let run = headers[1..=3].iter().copied().chain(forked).collect(); + + let err = c.apply(1, run).unwrap_err().to_string(); + assert!(err.contains("backfill"), "{err}"); + assert_eq!(c.header(5), Some(headers[5]), "chain is left untouched"); + assert_eq!(c.tip_height(), Some(10)); + } + + /// A set built for another network is not caught by anything else: here the headers are + /// real, so they link and hash correctly, and the difficulty rules are off on this network. + /// Genesis is the only block whose hash `params` fixes, so requiring it is what makes the + /// mismatch visible. + #[test] + fn rejects_a_trusted_set_without_genesis() { + let headers = mine(¶ms(), 6); + let err = HeaderChain::new(params(), [(5, headers[5])]) + .unwrap_err() + .to_string(); + assert!(err.contains("must include the genesis block"), "{err}"); + assert!(err.contains("starts at height 5"), "{err}"); + + // Empty is still fine: it means "sync from genesis", and has nothing to mismatch. + let c = HeaderChain::new(params(), []).unwrap(); + assert_eq!(c.base_height(), 1); + } + + #[test] + fn rejects_a_trusted_anchor_off_the_retarget_boundary() { + let params = retarget_params(); + let headers = mine(¶ms, 12); + let err = HeaderChain::new(params, [(0, headers[0]), (11, headers[11])]) + .unwrap_err() + .to_string(); + assert!(err.contains("difficulty-adjustment boundary"), "{err}"); + } + + #[test] + fn verifies_every_retarget_above_a_boundary_anchor() { + let params = retarget_params(); + let headers = mine(¶ms, 25); + assert_ne!( + headers[20].bits, headers[19].bits, + "difficulty must actually move for this test to mean anything" + ); + + let mut c = HeaderChain::new(params.clone(), [(0, headers[0]), (10, headers[10])]).unwrap(); + assert_eq!(c.base_height(), 11); + c.apply(11, headers[11..].to_vec()).unwrap(); + assert_eq!(c.tip_height(), Some(25)); + + // The retarget at 20 is recomputed from the trusted header at 10 — no gap on faith. + let mut faked = headers.clone(); + faked[20].bits = headers[19].bits; + let mut c = HeaderChain::new(params, [(0, headers[0]), (10, headers[10])]).unwrap(); + let err = c.apply(11, faked[11..].to_vec()).unwrap_err().to_string(); + assert!(err.contains("height 20"), "{err}"); + assert!(err.contains("consensus requires"), "{err}"); + } +} diff --git a/bdk_electrum_streaming/src/lib.rs b/bdk_electrum_streaming/src/lib.rs index fbaccc8..0e06a09 100644 --- a/bdk_electrum_streaming/src/lib.rs +++ b/bdk_electrum_streaming/src/lib.rs @@ -2,10 +2,16 @@ use std::collections::BTreeSet; -use bdk_core::{bitcoin::Txid, spk_client::FullScanResponse}; +use bdk_core::{ + bitcoin::{block::Header, Txid}, + spk_client::FullScanResponse, +}; +/// Re-export, so callers can inspect the errors this crate returns without matching its +/// `anyhow` version themselves. +pub use anyhow; +/// Re-export. pub use electrum_streaming_client; -use bdk_core::ConfirmationBlockTime; mod cache; pub use cache::*; mod state; @@ -26,9 +32,18 @@ mod blocking_client; pub use blocking_client::*; mod confirmation_job; pub use confirmation_job::*; +mod header_chain; +pub use header_chain::*; +mod anchor; +pub use anchor::*; -pub type Update = FullScanResponse; -pub type AnchorUpdate = BTreeSet<(ConfirmationBlockTime, Txid)>; +/// What a sync produces. +/// +/// Anchors are [`ProvenAnchor`]s — merkle-proved against a header in the verified +/// [`HeaderChain`] — and the chain update carries full [`Header`]s, so a block's time is read +/// from there rather than copied onto every anchor. +pub type Update = FullScanResponse; +pub type AnchorUpdate = BTreeSet<(ProvenAnchor, Txid)>; pub type BlockingClientAction = ClientAction>; pub type AsyncClientAction = ClientAction; diff --git a/bdk_electrum_streaming/src/spk_job.rs b/bdk_electrum_streaming/src/spk_job.rs index 0e62b36..37da75b 100644 --- a/bdk_electrum_streaming/src/spk_job.rs +++ b/bdk_electrum_streaming/src/spk_job.rs @@ -5,11 +5,11 @@ use std::{ use bdk_core::{ bitcoin::{OutPoint, Txid}, - ConfirmationBlockTime, TxUpdate, + TxUpdate, }; use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; -use crate::{req::ReqQueuer, Cache}; +use crate::{req::ReqQueuer, Cache, ProvenAnchor}; /// Where a [`SpkJob`] has got to. /// @@ -62,7 +62,7 @@ pub enum SpkProgress { Blocked, /// Everything asked for has arrived. Carries what the job gathered, leaving it empty, so a /// job polled again after finishing contributes nothing a second time. - Done(TxUpdate), + Done(TxUpdate), } /// The job to perform once we receive a script status notification. @@ -80,7 +80,9 @@ pub struct SpkJob { pub spk_hash: ElectrumScriptHash, stage: SpkStage, - tx_update: TxUpdate, + + /// Staged tx update. + tx_update: TxUpdate, } impl SpkJob { @@ -95,7 +97,7 @@ impl SpkJob { let stage = match spk_status { Some(status) => SpkStage::ProcessingHistory { status }, None => { - if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&spk_hash) { + if let Some(prev_txids) = cache.spk_txids.get(&spk_hash) { tx_update .evicted_ats .extend(prev_txids.iter().map(|&txid| (txid, start.as_secs()))); @@ -149,7 +151,7 @@ impl SpkJob { SpkStage::ProcessingHistory { status } => { match cache.subscriptions.spk_history(*status) { Some(history) => { - if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&self.spk_hash) { + if let Some(prev_txids) = cache.spk_txids.get(&self.spk_hash) { let these_txids = history.iter().map(|tx| tx.txid()).collect::>(); let to_evict = prev_txids @@ -176,7 +178,7 @@ impl SpkJob { } } SpkStage::ProcessingTxs(missing_txs) => { - missing_txs.retain(|txid| match cache.tx_cache.txs.get(txid) { + missing_txs.retain(|txid| match cache.txs.get(txid) { Some(tx) => { self.tx_update.txs.push(tx.clone()); false @@ -205,7 +207,7 @@ impl SpkJob { // `retain` cannot fail, so a bad output is carried out and raised below. let mut err = Option::::None; missing_prevouts.retain(|op| { - let tx = match cache.tx_cache.txs.get(&op.txid) { + let tx = match cache.txs.get(&op.txid) { Some(tx) => tx, None => { let txid = op.txid; diff --git a/bdk_electrum_streaming/src/state.rs b/bdk_electrum_streaming/src/state.rs index 648e925..5bdb389 100644 --- a/bdk_electrum_streaming/src/state.rs +++ b/bdk_electrum_streaming/src/state.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use anyhow::Context; -use bdk_core::{CheckPoint, ConfirmationBlockTime}; +use bdk_core::{bitcoin::block::Header, BlockId}; use electrum_streaming_client::{ notification::Notification, request, AsyncPendingRequest, BlockingPendingRequest, ElectrumScriptHash, ElectrumScriptStatus, MaybeBatch, PendingRequest, @@ -15,7 +15,7 @@ use crate::{ confirmation_job::{ConfirmationJob, ConfirmationProgress}, req::{JobRequest, PoppedRequest, ReqCoord, ReqQueue}, spk_job::{SpkJob, SpkProgress}, - DerivedSpkTracker, Update, + DerivedSpkTracker, HeaderChain, ProvenAnchor, Update, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -41,7 +41,7 @@ pub type BlockingState = State; pub struct State { spk_tracker: DerivedSpkTracker, coord: ReqCoord, - cp: CheckPoint, + chain: HeaderChain, cache: Cache, spk_jobs: BTreeMap, @@ -61,12 +61,12 @@ impl State { coord: ReqCoord, cache: Cache, spk_tracker: DerivedSpkTracker, - cp: CheckPoint, + chain: HeaderChain, ) -> Self { Self { spk_tracker, coord, - cp, + chain, cache, spk_jobs: BTreeMap::new(), confirmation_job: None, @@ -83,6 +83,11 @@ impl State { &self.cache.subscriptions } + /// Get a reference to the verified header chain. + pub fn chain(&self) -> &HeaderChain { + &self.chain + } + /// Insert a descriptor and queue outgoing requests (if needed). pub fn insert_descriptor( &mut self, @@ -163,7 +168,7 @@ impl State { self.spk_jobs.clear(); let update = core::mem::take(&mut self.staged); tracing::info!( - tip_height = self.cp.height(), + tip_height = self.chain.tip_height(), anchors = update.tx_update.anchors.len(), txs = update.tx_update.txs.len(), "Confirmation job finished" @@ -243,18 +248,26 @@ impl State { JobRequest::GetHeaders(req) => { let resp = from_raw(&req, raw)?; debug_assert!(job_ids.contains(&JobId::Confirmation)); - let blocks = self - .cache - .resolve_headers_query(req, resp) - .collect::>(); if let Some(job) = &mut self.confirmation_job { - job.resolve_blocks(blocks); + job.resolve_blocks((req.start_height..).zip(resp.headers)); } self.poll_confirmation_job(req_queue) } JobRequest::GetHistory(req) => { let resp = from_raw(&req, raw)?; - let resp_status = self.cache.resolve_history_query(req, resp); + let resp_status = ElectrumScriptStatus::from_history(&resp); + if let Some(spk_status) = resp_status { + self.cache + .spk_txids + .entry(req.script_hash) + .or_default() + .extend(resp.iter().map(|tx| tx.txid())); + self.cache + .subscriptions + .insert_spk(req.script_hash, spk_status, resp); + } else { + self.cache.subscriptions.remove_spk(req.script_hash); + } // A history that does not hash to the status a job awaits can never // satisfy it, and the two differing means the status moved — so a @@ -286,7 +299,7 @@ impl State { txid, )); } - self.cache.tx_cache.txs.insert(get_tx.txid, resp.tx.into()); + self.cache.txs.insert(get_tx.txid, resp.tx.into()); self.poll_spk_jobs(req_queue, job_ids)?; self.poll_confirmation_job(req_queue) } @@ -300,12 +313,12 @@ impl State { return self.poll_confirmation_job(req_queue); } - let cp = match self.cp.get(req.height) { - Some(cp) => cp, - // Not expected to fire: the job places every height before it asks - // a proof for it, and a height leaving the chain bumps the generation - // the check above catches. Getting here is our own bookkeeping - // breaking, not the server misbehaving. + let header = match self.chain.header(req.height) { + Some(header) => header, + // Not expected to fire: the job places every height before it asks a + // proof for it, and a height leaving the verified chain bumps the + // generation the check above catches. Getting here is our own + // bookkeeping breaking, not the server misbehaving. None => { debug_assert!( false, @@ -315,28 +328,7 @@ impl State { tracing::error!( ?req, ?resp, - "Received a merkle proof before we placed the block" - ); - self.confirmation_job = None; - return Ok(()); - } - }; - let header = match self.cache.headers.get(&cp.hash()) { - Some(header) => *header, - // Not expected either, and a reorg is not the reason — that is the - // check above. Every header a job puts in the chain lands in - // `Cache::headers` as it arrives, and nothing prunes them. - None => { - debug_assert!( - false, - "no header for {}, the block we hold at height {}", - cp.hash(), - req.height - ); - tracing::error!( - ?req, - blockhash = cp.hash().to_string(), - "No header for the block we hold at this height", + "Proof for a height the verified chain does not reach", ); self.confirmation_job = None; return Ok(()); @@ -350,11 +342,15 @@ impl State { block_hash = header.block_hash().to_string(), "Inserting anchor.", ); - self.cache.tx_cache.anchors.insert( + self.cache.anchors.insert( (req.txid, header.block_hash()), - ConfirmationBlockTime { - block_id: cp.block_id(), - confirmation_time: header.time as u64, + ProvenAnchor { + block_id: BlockId { + height: req.height, + hash: header.block_hash(), + }, + pos: resp.pos, + merkle: resp.merkle, }, ); } else { @@ -389,13 +385,8 @@ impl State { &mut self, req_queue: &mut ReqQueue, height: u32, - header: bdk_core::bitcoin::block::Header, + header: Header, ) -> anyhow::Result<()> { - // A same-height reorg is applied without fetching anything, so this announcement is the - // only place the replacement header is ever offered to us. Caching it here saves the - // anchor refetch a round-trip on the very path it exists for. - self.cache.headers.insert(header.block_hash(), header); - match &mut self.confirmation_job { Some(job) => { if job.set_tip(height, header) { @@ -429,7 +420,7 @@ impl State { self.cache.subscriptions.remove_spk(spk_hash); } - if spk_status.is_some() || self.cache.tx_cache.spk_txids.contains_key(&spk_hash) { + if spk_status.is_some() || self.cache.spk_txids.contains_key(&spk_hash) { for script_hash in self.spk_tracker.mark_script_hash_used(&k, i) { self.coord .queuer(req_queue, JobId::Spk(script_hash)) @@ -452,18 +443,17 @@ impl State { // A notification is all that revives a cancelled job, and below the reorg window the // tip never moves — so this is where an anchor the server has come back to is picked up. if self.confirmation_job.is_none() { - match self.cache.headers.get(&self.cp.hash()) { - Some(&header) => { - self.confirmation_job = Some(ConfirmationJob::new(self.cp.height(), header)); + match self.chain.tip() { + Some(cp) => { + self.confirmation_job = Some(ConfirmationJob::new(cp.height(), cp.data())); } - // Not expected to fire: a tip is only adopted through a notification, which - // caches its header. Ask for the tip rather than leave the anchors waiting on a - // block ten minutes out; a request already in flight absorbs this one. + // Not expected to fire: a tip is only adopted through a notification, and the + // very first one builds the chain's initial run before any spk status can + // arrive. Ask for the tip rather than leave the anchors waiting on a block ten + // minutes out; a request already in flight absorbs this one. None => { tracing::warn!( - tip_height = self.cp.height(), - tip_hash = self.cp.hash().to_string(), - "No header for our tip, so no confirmation job can be built. Resubscribing." + "No verified tip yet, so no confirmation job can be built. Resubscribing." ); self.coord .queuer(req_queue, JobId::Confirmation) @@ -550,7 +540,7 @@ impl State { loop { let progress = { let mut queuer = self.coord.queuer(req_queue, JobId::Confirmation); - match job.poll(&mut queuer, &self.cache, &self.cp) { + match job.poll(&mut queuer, &self.cache, &mut self.chain) { Ok(progress) => progress, Err(err) => { self.confirmation_job = Some(job); @@ -560,17 +550,16 @@ impl State { }; match progress { ConfirmationProgress::Continue => continue, - ConfirmationProgress::CheckPointUpdate { cp, evicted } => { - if !evicted.is_empty() { + ConfirmationProgress::ChainUpdate { cp, reorged } => { + if reorged { tracing::info!( - heights = ?evicted, - "Blocks evicted from the local chain. Refetching anchors." + tip_height = cp.height(), + "Blocks displaced from the verified chain. Refetching anchors." ); - // Responses to requests which are still in flight describe the chain we - // just left behind. + // Responses to requests which are still in flight were made against the + // chain we just left behind. self.coord.bump_chain_generation(); } - self.cp = cp.clone(); self.staged.chain_update = Some(cp); continue; } diff --git a/bdk_electrum_streaming/tests/env.rs b/bdk_electrum_streaming/tests/env.rs index 9b62998..0f11b73 100644 --- a/bdk_electrum_streaming/tests/env.rs +++ b/bdk_electrum_streaming/tests/env.rs @@ -1,18 +1,19 @@ use std::{sync::atomic::AtomicBool, time::Duration}; +use std::collections::BTreeMap; + use bdk_chain::{ - keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, CanonicalizationParams, - ChainPosition, IndexedTxGraph, + keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, ChainPosition, IndexedTxGraph, }; -use bdk_core::{ - bitcoin::{key::Secp256k1, params::REGTEST, Address, Amount, BlockHash, Txid}, - ConfirmationBlockTime, +use bdk_core::bitcoin::{ + block::Header, constants::genesis_block, key::Secp256k1, params::REGTEST, Address, Amount, + BlockHash, Network, Txid, }; use bdk_electrum_streaming::{ run_async, run_blocking, AsyncClient, AsyncState, BlockingClient, BlockingState, Cache, - DerivedSpkTracker, ReqCoord, Update, + DerivedSpkTracker, HeaderChain, ProvenAnchor, ReqCoord, Update, }; -use bdk_testenv::{bitcoincore_rpc::RpcApi, utils::DESCRIPTORS, TestEnv}; +use bdk_testenv::{electrsd::electrum_client::ElectrumApi, utils::DESCRIPTORS, TestEnv}; use futures::{channel::mpsc, pin_mut, FutureExt, StreamExt}; use miniscript::Descriptor; use tokio::net::TcpStream; @@ -23,6 +24,8 @@ const EXTERNAL: &str = "external"; const INTERNAL: &str = "internal"; const LOOKAHEAD: u32 = 6; +type Graph = IndexedTxGraph>; + fn init() { let _ = tracing_subscriber::fmt() .with_test_writer() @@ -30,51 +33,76 @@ fn init() { .try_init(); } +fn genesis_header() -> Header { + genesis_block(®TEST).header +} + fn apply_update( - chain: &mut LocalChain, - graph: &mut IndexedTxGraph>, + chain: &mut LocalChain
, + graph: &mut Graph, update: Update<&'static str>, ) -> anyhow::Result<()> { let _ = graph .index .reveal_to_target_multi(&update.last_active_indices); let _ = graph.apply_update(update.tx_update); - if let Some(cp) = update.chain_update { - chain.apply_update(cp)?; - } else { - panic!("NO CHAIN UPDATE!"); - } + let cp = update.chain_update.expect("NO CHAIN UPDATE!"); + chain.apply_update(cp)?; Ok(()) } -#[test] -fn blocking_env() -> anyhow::Result<()> { - init(); - +/// Set up an indexer/graph/chain trio plus the spk tracker fed to the client. +fn setup() -> anyhow::Result<(Graph, LocalChain
, DerivedSpkTracker<&'static str>)> { let secp = Secp256k1::new(); - let env = TestEnv::new()?; - let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; + let (external, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; + let (internal, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - let mut graph = IndexedTxGraph::::new({ + let graph = IndexedTxGraph::new({ let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); indexer.insert_descriptor(EXTERNAL, external.clone())?; indexer.insert_descriptor(INTERNAL, internal.clone())?; indexer }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); + let (chain, _) = LocalChain::from_genesis(genesis_header()); let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); spk_tracker.insert_descriptor(EXTERNAL, external, 0); spk_tracker.insert_descriptor(INTERNAL, internal, 0); + Ok((graph, chain, spk_tracker)) +} + +fn confirmed_balance(chain: &LocalChain
, graph: &Graph) -> bdk_chain::Balance { + // `LocalChain` only canonicalizes over blockhashes for now, so drop the headers. + let chain = LocalChain::from_blocks( + chain + .iter_checkpoints() + .map(|cp| (cp.height(), cp.hash())) + .collect::>(), + ) + .expect("must build blockhash chain"); + chain + .canonical_view(graph.graph(), chain.tip().block_id(), Default::default()) + .balance( + graph.index.outpoints().clone(), + |(k, _), _| *k == INTERNAL, + 0, + ) +} + +#[test] +fn blocking_env() -> anyhow::Result<()> { + init(); + + let env = TestEnv::new()?; + let electrum_url = env.electrsd.electrum_url.clone(); + let (mut graph, mut chain, spk_tracker) = setup()?; + let mut state = BlockingState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, update_rx) = std::sync::mpsc::channel::>(); @@ -112,17 +140,7 @@ fn blocking_env() -> anyhow::Result<()> { } } - let balance = graph.graph().balance( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - graph.index.outpoints().clone(), - |(k, _), _| *k == INTERNAL, - ); - for cp in chain.iter_checkpoints() { - println!("height={}, hash={}", cp.height(), cp.hash()); - } - println!("BALANCE: {}", balance); + println!("BALANCE: {}", confirmed_balance(&chain, &graph)); // TODO: Figure out a way to stop the thread without having to close the connection. conn.shutdown(std::net::Shutdown::Both)?; @@ -136,30 +154,15 @@ fn blocking_env() -> anyhow::Result<()> { async fn env() -> anyhow::Result<()> { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); - - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + let (mut graph, mut chain, spk_tracker) = setup()?; let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -208,17 +211,7 @@ async fn env() -> anyhow::Result<()> { } } - let balance = graph.graph().balance( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - graph.index.outpoints().clone(), - |(k, _), _| *k == INTERNAL, - ); - for cp in chain.iter_checkpoints() { - println!("height={}, hash={}", cp.height(), cp.hash()); - } - println!("BALANCE: {}", balance); + println!("BALANCE: {}", confirmed_balance(&chain, &graph)); client.stop().await?; run_handle.await??; @@ -226,38 +219,45 @@ async fn env() -> anyhow::Result<()> { Ok(()) } -/// A new block confirming a tracked tx must anchor it on the live connection — no reconnect, -/// regardless of the order in which the server sends the script hash and header notifications. -/// The order-sensitive case (history reporting a height above the local tip) is pinned -/// deterministically in `tests/state.rs`. +/// Trust only the tip, so the wallet's whole history sits below the sync start and can only be +/// anchored by backfilling headers from a trusted block below it. #[tokio::test] -async fn new_block_confirmation_is_anchored_live() -> anyhow::Result<()> { +async fn backfills_history_below_the_sync_start() -> anyhow::Result<()> { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); + let (mut graph, mut chain, spk_tracker) = setup()?; - let (external, _external_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _internal_keys) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); + // Mine everything *before* the client ever connects. + let ((_, spk), _) = graph + .index + .next_unused_spk(EXTERNAL) + .expect("must derive spk"); + env.mine_blocks(101, Some(Address::from_script(&spk, ®TEST)?))?; + env.wait_until_electrum_sees_block(Duration::from_secs(30))?; - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + // The user vouches for this block; take the header from the server but hold it to the hash. + let trusted_height = 101_u32; + let trusted_header = env + .electrum_client() + .block_header(trusted_height as usize)?; + let trusted_hash: BlockHash = env.get_block_hash(trusted_height as u64)?; + assert_eq!(trusted_header.block_hash(), trusted_hash); let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new( + Network::Regtest, + [(0, genesis_header()), (trusted_height, trusted_header)], + )?, + ); + assert_eq!( + state.chain().base_height(), + trusted_height + 1, + "sync starts one block above the highest trusted block", ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -277,113 +277,65 @@ async fn new_block_confirmation_is_anchored_live() -> anyhow::Result<()> { anyhow::Ok(()) }); - let update = update_rx.next().await.expect("Must have next update"); - apply_update(&mut chain, &mut graph, update)?; - - // Coinbase maturity, so that `env.send` has funds to spend. - env.mine_blocks(101, None)?; - let premine_height = env.rpc_client().get_block_count()? as u32; - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); pin_mut!(timeout); - while chain.tip().height() < premine_height { - futures::select! { - _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for chain sync.")), - update = update_rx.next() => { - let update = update.expect("Must have next update"); - apply_update(&mut chain, &mut graph, update)?; - }, - } - } - - let ((_, spk), _) = graph - .index - .next_unused_spk(EXTERNAL) - .expect("must derive spk"); - let txid = env.send(&Address::from_script(&spk, ®TEST)?, Amount::ONE_BTC)?; - - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); - pin_mut!(timeout); - loop { - futures::select! { - _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for unconfirmed tx.")), - update = update_rx.next() => { - let update = update.expect("Must have next update"); - let has_tx = update.tx_update.txs.iter().any(|tx| tx.compute_txid() == txid); - apply_update(&mut chain, &mut graph, update)?; - if has_tx { - break; - } - }, - } - } - let confirm_height = env.rpc_client().get_block_count()? as u32 + 1; - env.mine_blocks(1, None)?; - - let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); - pin_mut!(timeout); let anchor = loop { futures::select! { - _ = timeout => return Err(anyhow::anyhow!( - "Timed-out waiting for anchor: tx is still unconfirmed at tip height {}", - chain.tip().height(), - )), + _ = timeout => return Err(anyhow::anyhow!("Timed-out waiting for the backfill.")), update = update_rx.next() => { let update = update.expect("Must have next update"); + let anchor = update.tx_update.anchors.iter().next().cloned(); apply_update(&mut chain, &mut graph, update)?; - let confirmed_anchor = graph - .graph() - .list_canonical_txs( - &chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - ) - .find(|ctx| ctx.tx_node.txid == txid) - .and_then(|ctx| match ctx.chain_position { - ChainPosition::Confirmed { anchor, .. } => Some(anchor), - ChainPosition::Unconfirmed { .. } => None, - }); - if let Some(anchor) = confirmed_anchor { + if let Some((anchor, _)) = anchor { break anchor; } }, } }; - assert_eq!(anchor.block_id.height, confirm_height); + + assert!( + anchor.block_id.height <= trusted_height, + "the coinbase we found is below the sync start, at {}", + anchor.block_id.height, + ); + assert!( + !anchor.merkle.is_empty() || anchor.pos == 0, + "the anchor carries its merkle proof", + ); + assert_eq!( + chain.get(anchor.block_id.height).map(|cp| cp.hash()), + Some(anchor.block_id.hash), + "the backfilled header made it into the chain update", + ); + assert!(confirmed_balance(&chain, &graph).confirmed.to_sat() > 0); client.stop().await?; run_handle.await??; - Ok(()) } -type Graph = IndexedTxGraph>; - -/// The anchor `txid` is canonically confirmed at, if it is confirmed at all. -fn canonical_anchor( - chain: &LocalChain, - graph: &Graph, - txid: Txid, -) -> Option { - graph - .graph() - .list_canonical_txs( - chain, - chain.tip().block_id(), - CanonicalizationParams::default(), - ) - .find(|ctx| ctx.tx_node.txid == txid) - .and_then(|ctx| match ctx.chain_position { - ChainPosition::Confirmed { anchor, .. } => Some(anchor), - ChainPosition::Unconfirmed { .. } => None, - }) +/// The anchor a canonical, confirmed `txid` has — `None` if it is unconfirmed or unknown. +fn canonical_anchor(chain: &LocalChain
, graph: &Graph, txid: Txid) -> Option { + // `LocalChain` only canonicalizes over blockhashes for now, so drop the headers. + let chain = LocalChain::from_blocks( + chain + .iter_checkpoints() + .map(|cp| (cp.height(), cp.hash())) + .collect::>(), + ) + .expect("must build blockhash chain"); + let view = chain.canonical_view(graph.graph(), chain.tip().block_id(), Default::default()); + match view.tx(txid)?.pos { + ChainPosition::Confirmed { anchor, .. } => Some(anchor), + ChainPosition::Unconfirmed { .. } => None, + } } /// A live client against a fresh `electrsd`, with the machinery the reorg tests share. struct LiveWallet { env: TestEnv, - chain: LocalChain, + chain: LocalChain
, graph: Graph, update_rx: mpsc::UnboundedReceiver>, client: AsyncClient<&'static str>, @@ -395,30 +347,15 @@ impl LiveWallet { async fn new() -> anyhow::Result { init(); - let secp = Secp256k1::new(); let env = TestEnv::new()?; let electrum_url = env.electrsd.electrum_url.clone(); - - let (external, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; - let (internal, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; - - let mut graph = IndexedTxGraph::::new({ - let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); - indexer.insert_descriptor(EXTERNAL, external.clone())?; - indexer.insert_descriptor(INTERNAL, internal.clone())?; - indexer - }); - let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); - - let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); - spk_tracker.insert_descriptor(EXTERNAL, external, 0); - spk_tracker.insert_descriptor(INTERNAL, internal, 0); + let (mut graph, mut chain, spk_tracker) = setup()?; let mut state = AsyncState::new( ReqCoord::default(), Cache::default(), spk_tracker, - chain.tip(), + HeaderChain::new(Network::Regtest, [(0, genesis_header())])?, ); let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); @@ -451,6 +388,10 @@ impl LiveWallet { }) } + fn block_count(&self) -> anyhow::Result { + Ok(self.env.rpc_client().get_block_count()?.0 as u32) + } + /// Apply updates until `f` holds. /// /// Errors if the client stops — which is what a connection torn down by an expected server @@ -458,7 +399,7 @@ impl LiveWallet { async fn wait_until( &mut self, what: &str, - mut f: impl FnMut(&LocalChain, &Graph) -> bool, + mut f: impl FnMut(&LocalChain
, &Graph) -> bool, ) -> anyhow::Result<()> { let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); pin_mut!(timeout); @@ -483,7 +424,7 @@ impl LiveWallet { /// Returns the txid and the hash of the block confirming it. async fn confirm_tracked_tx(&mut self) -> anyhow::Result<(Txid, BlockHash)> { self.env.mine_blocks(101, None)?; - let premine_height = self.env.rpc_client().get_block_count()? as u32; + let premine_height = self.block_count()?; self.wait_until("the premined chain", |chain, _| { chain.tip().height() >= premine_height }) @@ -522,34 +463,29 @@ impl LiveWallet { /// Issue #12, end to end: a reorg re-mines a confirmed tx into a *different* block at the *same* /// height. The Electrum script status is a hash over txid-height pairs, so it is unchanged and no /// script hash notification is sent. The anchor must still be refetched off the tip alone. +/// +/// The fork also has to out-work the chain it replaces, or the verified chain will not take it — +/// so the replacement run is one block longer, which is what makes a real node switch too. #[tokio::test] async fn reorg_to_same_height_block_refetches_anchor_live() -> anyhow::Result<()> { let mut w = LiveWallet::new().await?; let (txid, first_block) = w.confirm_tracked_tx().await?; - let confirm_height = w.env.rpc_client().get_block_count()? as u32; + let confirm_height = w.block_count()?; // Invalidate the confirming block and re-mine at the same height. The tx is back in the - // mempool, so it goes into the replacement block too. + // mempool, so it goes into the replacement block too. One extra block gives the fork the + // work it needs to be adopted. w.env.reorg(1)?; - let second_block = w.env.rpc_client().get_best_block_hash()?; + w.env.mine_empty_block()?; + let second_block = w + .env + .rpc_client() + .get_block_hash(confirm_height as u64)? + .block_hash()?; assert_ne!( first_block, second_block, "the reorg must actually replace the block" ); - assert_eq!( - w.env.rpc_client().get_block_count()? as u32, - confirm_height, - "the replacement block must be at the same height" - ); - assert!( - w.env - .rpc_client() - .get_block(&second_block)? - .txdata - .iter() - .any(|tx| tx.compute_txid() == txid), - "the replacement block must still contain the tx" - ); w.wait_until("the refetched anchor", |chain, graph| { canonical_anchor(chain, graph, txid).is_some_and(|a| a.block_id.hash == second_block) @@ -569,19 +505,15 @@ async fn reorg_to_same_height_block_refetches_anchor_live() -> anyhow::Result<() async fn reorg_unconfirming_a_tx_keeps_the_connection_alive() -> anyhow::Result<()> { let mut w = LiveWallet::new().await?; let (txid, _) = w.confirm_tracked_tx().await?; - let confirm_height = w.env.rpc_client().get_block_count()? as u32; + let confirm_height = w.block_count()?; // Invalidate the confirming block and replace it with empty ones, so the tx cannot be // re-mined and the server has no proof to give at that height. w.env.invalidate_blocks(1)?; w.env.mine_empty_block()?; w.env.mine_empty_block()?; - let tip_height = w.env.rpc_client().get_block_count()? as u32; + let tip_height = w.block_count()?; assert_eq!(tip_height, confirm_height + 1); - assert!( - w.env.rpc_client().get_raw_mempool()?.contains(&txid), - "the tx must be back in the mempool, so the server really has no proof for it" - ); // The connection has to keep serving: `wait_until` fails if the client stops. w.wait_until("the chain tip after the reorg", |chain, _| { diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index 7bc28a0..8705026 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -9,13 +9,13 @@ use bdk_core::{ transaction, Amount, CompactTarget, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Txid, Witness, }, - BlockId, CheckPoint, ConfirmationBlockTime, + BlockId, }; use bdk_electrum_streaming::{ electrum_streaming_client::{ response, ElectrumScriptHash, ElectrumScriptStatus, RawNotificationOrResponse, RawRequest, }, - BlockingState, Cache, DerivedSpkTracker, ReqCoord, ReqQueue, Update, + BlockingState, Cache, DerivedSpkTracker, HeaderChain, ProvenAnchor, ReqCoord, ReqQueue, Update, }; use miniscript::{Descriptor, DescriptorPublicKey}; use serde_json::json; @@ -152,6 +152,21 @@ fn drain_requests( updates } +/// Drain like [`drain_requests`], but hand back the first error rather than panicking on it. +fn drain_requests_fallible( + state: &mut BlockingState, + queue: &mut ReqQueue, + server: &Server, +) -> anyhow::Result>> { + let mut updates = Vec::new(); + while let Some(req) = queue.pop_front() { + if let Some(update) = state.poll(queue, response(&req, server))? { + updates.push(update); + } + } + Ok(updates) +} + /// Drain like [`drain_requests`], but answer every merkle proof in the queue ahead of everything /// else in each round. /// @@ -222,17 +237,28 @@ fn tx_paying(spk: &ScriptBuf, sats: u64) -> Transaction { } } +/// Grind `nonce` upwards until the header actually clears its own target. +/// +/// Regtest's target is easy but not free: a little under half of all nonces miss it, and the +/// header chain checks proof-of-work for real. +fn mine(mut header: block::Header) -> block::Header { + while header.validate_pow(header.target()).is_err() { + header.nonce = header.nonce.wrapping_add(1); + } + header +} + /// The regtest genesis block and an empty block on top of it. fn base_headers() -> (block::Header, block::Header) { let genesis = constants::genesis_block(Network::Regtest).header; - let header_1 = block::Header { + let header_1 = mine(block::Header { version: block::Version::ONE, prev_blockhash: genesis.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 100, bits: CompactTarget::from_consensus(0x207fffff), nonce: 0, - }; + }); (genesis, header_1) } @@ -243,14 +269,14 @@ fn block_with_root( time: u32, nonce: u32, ) -> block::Header { - block::Header { + mine(block::Header { version: block::Version::ONE, prev_blockhash: prev.block_hash(), merkle_root, time, bits: CompactTarget::from_consensus(0x207fffff), nonce, - } + }) } /// A block whose only transaction is `txid`, so that its merkle root is the txid itself. @@ -258,39 +284,46 @@ fn block_with_tx(prev: &block::Header, txid: Txid, time: u32, nonce: u32) -> blo block_with_root(prev, Txid::to_raw_hash(txid).into(), time, nonce) } -fn new_state( - cache: Cache, - descriptor: Descriptor, - genesis: block::Header, -) -> BlockingState { - new_state_with_cp( - cache, - descriptor, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), - ) +/// A state that trusts nothing but genesis, so everything above it has to be verified. +fn new_state(cache: Cache, descriptor: Descriptor) -> BlockingState { + new_state_trusting(cache, descriptor, []) } -fn new_state_with_cp( +fn new_state_trusting( cache: Cache, descriptor: Descriptor, - cp: CheckPoint, + trusted: impl IntoIterator, ) -> BlockingState { let mut spk_tracker = DerivedSpkTracker::new(0); spk_tracker.insert_descriptor("external", descriptor, 0); - BlockingState::new(ReqCoord::default(), cache, spk_tracker, cp) + // Genesis goes in here rather than at every call site: a non-empty trusted set has to carry + // it, and an empty one is unchanged by it. + let genesis = constants::genesis_block(Network::Regtest).header; + let chain = HeaderChain::new(Network::Regtest, trusted.into_iter().chain([(0, genesis)])) + .expect("must build header chain"); + BlockingState::new(ReqCoord::default(), cache, spk_tracker, chain) } -/// The anchor a tx confirmed in `header` at `height` must be given. -fn anchor_of(header: &block::Header, height: u32) -> ConfirmationBlockTime { - ConfirmationBlockTime { +/// The anchor a tx confirmed in `header` at `height` must be given, for a server answering with +/// the empty proof that a single-transaction block has. +fn anchor_of(header: &block::Header, height: u32) -> ProvenAnchor { + anchor_proved_by(header, height, Vec::new(), 0) +} + +/// The same, for a server answering with a real merkle branch. +fn anchor_proved_by( + header: &block::Header, + height: u32, + merkle: Vec, + pos: usize, +) -> ProvenAnchor { + ProvenAnchor { block_id: BlockId { height, hash: header.block_hash(), }, - confirmation_time: header.time as u64, + pos, + merkle, } } @@ -308,9 +341,9 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( let header_2 = block_with_tx(&header_1, txid, 200, 0); let mut cache = Cache::default(); - cache.tx_cache.txs.insert(txid, Arc::new(tx.clone())); + cache.txs.insert(txid, Arc::new(tx.clone())); - let mut state = new_state(cache, descriptor, genesis); + let mut state = new_state(cache, descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1], @@ -373,15 +406,11 @@ fn descriptor_inserted_mid_connection_is_subscribed() -> anyhow::Result<()> { let descriptor = Descriptor::::from_str(&format!("wpkh({XPUB}/0/*)"))?; let spk_hash = ElectrumScriptHash::new(descriptor.at_derivation_index(0)?.script_pubkey()); - let genesis = constants::genesis_block(Network::Regtest).header; let mut state = BlockingState::new( ReqCoord::default(), Cache::default(), DerivedSpkTracker::new(0), - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, [])?, ); let mut queue = ReqQueue::new(); @@ -422,10 +451,7 @@ fn last_active_index_is_index_of_active_spk() -> anyhow::Result<()> { ReqCoord::default(), Cache::default(), spk_tracker, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, []).expect("must build header chain"), ); let mut queue = ReqQueue::new(); let server = Server { @@ -480,10 +506,7 @@ fn last_active_index_is_highest_regardless_of_notification_order() -> anyhow::Re ReqCoord::default(), Cache::default(), spk_tracker, - CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }), + HeaderChain::new(Network::Regtest, []).expect("must build header chain"), ); let mut queue = ReqQueue::new(); let mut server = Server { @@ -550,14 +573,10 @@ fn anchor_is_refetched_when_tx_moves_to_another_block_of_same_height() -> anyhow let header_2 = block_with_tx(&header_1, txid, 200, 0); // The block that replaces height 2 contains the tx too, hence the identical script status. let header_2b = block_with_tx(&header_1, txid, 222, 1); - let header_3b = block::Header { - prev_blockhash: header_2b.block_hash(), - time: 300, - ..header_1 - }; + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); assert_ne!(header_2.block_hash(), header_2b.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -626,7 +645,7 @@ fn merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let header_2b = block_with_root(&header_1, proof_2b.expected_merkle_root(txid), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -673,10 +692,13 @@ fn merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let anchors = updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>(); assert!( - anchors.contains(&(anchor_of(&header_2b, 2), txid)), + anchors.contains(&( + anchor_proved_by(&header_2b, 2, proof_2b.merkle.clone(), proof_2b.pos), + txid + )), "the anchor must be refetched rather than written off from a proof of the evicted block" ); Ok(()) @@ -700,7 +722,7 @@ fn anchors_staged_before_a_reorg_are_not_emitted_after_it() -> anyhow::Result<() let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -742,7 +764,7 @@ fn anchors_staged_before_a_reorg_are_not_emitted_after_it() -> anyhow::Result<() let anchors = updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>(); for evicted in [ (anchor_of(&header_2, 2), txid_a), @@ -780,7 +802,7 @@ fn a_tx_unconfirmed_by_a_reorg_does_not_error_the_connection() -> anyhow::Result let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 333, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -834,7 +856,7 @@ fn merkle_error_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::R let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -910,7 +932,7 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -943,7 +965,7 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { "the job must give up on the pair rather than re-ask" ); assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "an error proves nothing, so no anchor may be recorded from it" ); @@ -970,12 +992,16 @@ fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { Ok(()) } -/// Issue #12's literal case: a reorg to a block of the *same* height, with no growth at all. -/// The tip announcement carries the replacement header, so this is the one reorg shape that can -/// be applied without fetching a single block — and every other reorg test here also grows the -/// chain, which takes a different path. +/// A fork of the same height, carrying no more work than the chain it would replace, must be +/// refused — and refusing it must leave the anchor we already have intact. +/// +/// This is what a full node does: an equal-work fork loses to the chain already in hand. The +/// server having moved to it is not evidence, since a server is exactly what the verified chain +/// exists to stop trusting. When the fork does out-work us, the tip announcement that says so is +/// what triggers the switch, and +/// [`anchor_is_refetched_when_tx_moves_to_another_block_of_same_height`] covers that. #[test] -fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { +fn a_fork_without_more_work_is_refused() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); @@ -984,7 +1010,7 @@ fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { let header_2b = block_with_tx(&header_1, txid, 222, 1); assert_ne!(header_2.block_hash(), header_2b.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1004,22 +1030,25 @@ fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { // The tip does not move: same height, different block, unchanged script status. server.headers = vec![genesis, header_1, header_2b]; - state.poll( + let notified = state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", "method": "blockchain.headers.subscribe", "params": [{ "hex": serialize_hex(&header_2b), "height": 2 }], })), - )?; - let updates = drain_requests(&mut state, &mut queue, &server); - + ); + let err = notified + .and_then(|_| drain_requests_fallible(&mut state, &mut queue, &server)) + .expect_err("an equal-work fork must be refused"); assert!( - updates.iter().any(|u| u - .tx_update - .anchors - .contains(&(anchor_of(&header_2b, 2), txid))), - "anchor must be refetched for the block that replaced the evicted one" + format!("{err:#}").contains("without more work"), + "the error must name the reason, got: {err:#}" + ); + assert_eq!( + state.chain().block_hash(2), + Some(header_2.block_hash()), + "the chain must be left on the block it already had" ); Ok(()) } @@ -1039,7 +1068,7 @@ fn anchor_is_refetched_whatever_order_the_server_answers_in() -> anyhow::Result< let header_2b = block_with_tx(&header_1, txid, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1078,16 +1107,14 @@ fn anchor_is_refetched_whatever_order_the_server_answers_in() -> anyhow::Result< Ok(()) } -/// A client restored from a persisted checkpoint chain starts with blocks in its chain whose -/// headers are not in its cache. Resolving an anchor at such a height needs both the header and -/// the proof, and nothing else will fetch that header — the chain consistency pass has nothing to -/// do, the tip being already correct. +/// A client restored from a trusted block has no verified history below it, so a transaction +/// confirmed down there cannot be anchored until the chain is backfilled to a block it trusts. /// -/// So this is the case where the two halves of the ordering fix are load-bearing: the proof must -/// not be asked for before the header is cached, and the header response must poll the waiting -/// job even though the chain itself has nothing to learn from it. +/// The proof must still not be checked before the header it is verified against arrives, which +/// `drain_requests_proofs_first` forces by answering every merkle request ahead of everything +/// else. #[test] -fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::Result<()> { +fn anchor_below_the_trusted_block_is_backfilled() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); @@ -1095,23 +1122,9 @@ fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::R let header_2 = block_with_tx(&header_1, txid, 200, 0); let header_3 = block_with_root(&header_2, TxMerkleNode::all_zeros(), 300, 0); - // The restored chain knows the blocks, the fresh cache knows none of their headers. The tx - // is one block below the tip, so the header it needs is not the one `headers.subscribe` - // hands back, and the chain consistency pass has nothing to do either, the tip being - // already correct. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: header_2.block_hash(), - }) - .insert(BlockId { - height: 3, - hash: header_3.block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + // Trusting the tip puts the chain base above the transaction, so nothing below it is + // verified and the sync range covers none of it. + let mut state = new_state_trusting(Cache::default(), descriptor, [(3, header_3)]); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -1127,18 +1140,22 @@ fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::R .tx_update .anchors .contains(&(anchor_of(&header_2, 2), txid))), - "the anchor must resolve even when the proof overtakes the header it is verified against" + "the anchor must resolve once history below the trusted block is backfilled" + ); + assert_eq!( + state.chain().base_height(), + 1, + "the chain must have grown down to just above genesis" ); Ok(()) } /// A header batch fetched before a reorg describes the chain we have since left behind, and -/// splicing it in would put a purged block into the checkpoint chain. +/// splicing it in would put a purged block into the verified chain. /// -/// The case that exposes it is a *sparse* chain — a restored one, or one whose missing heights -/// sit below the reorg window `ConfirmationJob` rewrites — reorged deeper than that window, so the -/// consistency pass never learns the low block changed too. Only the height the anchor needs -/// brings it back, and that fetch was in flight when the chain moved. +/// The batch is dropped when the tip that wanted it is abandoned, so the answer never reaches +/// the chain at all — and even if it did, it neither links to the chain we moved to nor +/// out-works it. #[test] fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; @@ -1146,8 +1163,9 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - // Two chains which differ at height 2 as well as near the tip. Both contain the tx at - // height 2, so the anchor stays valid throughout — only the block it belongs to changes. + // Two chains which differ at height 2. Both contain the tx there, so the anchor stays valid + // throughout — only the block it belongs to changes. The second is longer, so it carries + // more work and is allowed to replace the first. let build = |second: block::Header, tip: u32, nonce: u32| { let mut chain = vec![genesis, header_1, second]; for height in 3..=tip { @@ -1161,22 +1179,12 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu } chain }; - let chain_a = build(block_with_tx(&header_1, txid, 200, 0), 30, 0); - let chain_b = build(block_with_tx(&header_1, txid, 222, 1), 31, 1); + let chain_a = build(block_with_tx(&header_1, txid, 200, 0), 8, 0); + let chain_b = build(block_with_tx(&header_1, txid, 222, 1), 9, 1); let (a2, b2) = (chain_a[2], chain_b[2]); assert_ne!(a2.block_hash(), b2.block_hash()); - // A restored chain sparse enough that height 2 is a gap — so the anchor has to fetch that - // header, and `replaces` will not decline it when it comes back. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain_a[30].block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain_a.clone(), @@ -1204,15 +1212,13 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu }; let stale_resp = response(&stale_req, &server); - // The reorg lands. It runs deeper than the reorg window, so the consistency pass rewrites - // only the top of the chain and never learns that height 2 changed too. server.headers = chain_b.clone(); state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", "method": "blockchain.headers.subscribe", - "params": [{ "hex": serialize_hex(&chain_b[31]), "height": 31 }], + "params": [{ "hex": serialize_hex(&chain_b[9]), "height": 9 }], })), )?; let mut updates = drain_requests(&mut state, &mut queue, &server); @@ -1221,21 +1227,15 @@ fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Resu updates.extend(state.poll(&mut queue, stale_resp)?); updates.extend(drain_requests(&mut state, &mut queue, &server)); - let tip = updates - .iter() - .rev() - .find_map(|u| u.chain_update.clone()) - .expect("must get a chain update"); - let at_2 = tip.iter().find(|cp| cp.height() == 2); - assert_ne!( - at_2.as_ref().map(|cp| cp.hash()), - Some(a2.block_hash()), - "a header from the chain we left must not be spliced into the checkpoint chain" - ); assert_eq!( - at_2.map(|cp| cp.hash()), + state.chain().block_hash(2), Some(b2.block_hash()), - "the height must be refetched against the chain we are actually on" + "the height must be verified against the chain we are actually on" + ); + assert_ne!( + state.chain().block_hash(2), + Some(a2.block_hash()), + "a header from the chain we left must not enter the verified chain" ); assert!( updates @@ -1262,7 +1262,7 @@ fn a_replayed_job_does_not_displace_one_the_server_started() -> anyhow::Result<( let header_2b = block_with_tx(&header_1, txid_a, 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1341,24 +1341,25 @@ fn a_replayed_job_does_not_displace_one_the_server_started() -> anyhow::Result<( Ok(()) } -/// A persisted checkpoint chain can be stale at a height below the window `ConfirmationJob` rewrites -/// — an offline reorg, say. Then the block *we* have at that height is one the server does not -/// have, and a header request is keyed by height, so no request can ever fetch it. +/// A trusted block the server disagrees with must stop the connection, not spin it. /// -/// Withholding the proof until that header is cached must not turn into an endless request loop. +/// Trusting a block is the user's assertion that it is canonical, so a server offering a +/// different one at that height is not something to reconcile — every header above it descends +/// from a block we do not accept. The old client would ask again forever; this one refuses the +/// run and errors out, and the error names the conflict. #[test] -fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { +fn a_server_disagreeing_with_a_trusted_block_stops_the_connection() -> anyhow::Result<()> { let (descriptor, _spk_hash, spk) = tracked_descriptor()?; let tx = tx_paying(&spk, 50_000); let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - // The server's height 2, and the stale one our persisted chain still claims. + // The server's height 2, and the one we have been told to trust. let server_2 = block_with_tx(&header_1, txid, 200, 0); - let stale_2 = block_with_tx(&header_1, txid, 999, 7); - assert_ne!(server_2.block_hash(), stale_2.block_hash()); + let trusted_2 = block_with_tx(&header_1, txid, 999, 7); + assert_ne!(server_2.block_hash(), trusted_2.block_hash()); let mut chain = vec![genesis, header_1, server_2]; - for height in 3..=30u32 { + for height in 3..=8u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( &prev, @@ -1368,20 +1369,7 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { )); } - // Tip agrees with the server, so the consistency pass never rewrites height 2. - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: stale_2.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut state = new_state_trusting(Cache::default(), descriptor, [(2, trusted_2)]); let mut queue = ReqQueue::new(); let server = Server { headers: chain, @@ -1391,7 +1379,11 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { state.start(&mut queue); let mut served = 0; - while let Some(req) = queue.pop_front() { + let err = loop { + let req = match queue.pop_front() { + Some(req) => req, + None => panic!("the client must not settle while it disagrees with the server"), + }; served += 1; assert!( served < 200, @@ -1399,8 +1391,15 @@ fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { req.method, req.params ); - state.poll(&mut queue, response(&req, &server))?; - } + if let Err(err) = state.poll(&mut queue, response(&req, &server)) { + break err; + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("does not link") || msg.contains("trusted"), + "the error must name the conflict, got: {msg}" + ); Ok(()) } @@ -1420,7 +1419,7 @@ fn a_history_that_comes_back_empty_clears_the_subscription() -> anyhow::Result<( let (genesis, header_1) = base_headers(); let header_2 = block_with_tx(&header_1, txid, 200, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1495,7 +1494,7 @@ fn a_script_whose_history_goes_away_is_not_replayed() -> anyhow::Result<()> { let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2], @@ -1578,8 +1577,11 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { let theirs = block_with_root(&header_1, proof_theirs.expected_merkle_root(txid), 222, 1); assert_ne!(ours.merkle_root, theirs.merkle_root); - let mut chain = vec![genesis, header_1, theirs]; - for height in 3..=30u32 { + // The chain the server serves is ours — a block it genuinely disagreed with would be caught + // by header verification long before any proof. What it gets wrong is the *proof*: the + // branch it answers with expands to a root that is not the one in our block. + let mut chain = vec![genesis, header_1, ours]; + for height in 3..=8u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( &prev, @@ -1589,24 +1591,7 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { )); } - // A persisted chain holding our block at height 2, and its header already cached — otherwise - // `GetHeader` catches the disagreement before any proof is asked for. The tip agrees, so no - // chain job runs and nothing rewrites height 2: the disagreement is below the window. - let mut cache = Cache::default(); - cache.headers.insert(ours.block_hash(), ours); - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: ours.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(cache, descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain, @@ -1622,13 +1607,12 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { state.poll(&mut queue, response(&req, &server))?; } assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "a proof for a block we do not have must not anchor anything" ); - // The server comes back to our block at that height. Nothing durable was written against it, - // so the job a notification rebuilds must be able to anchor there. - server.headers[2] = ours; + // The server starts answering with the right proof. Nothing durable was written against the + // bad one, so the job a notification rebuilds must be able to anchor there. server.merkle_proof = (Vec::new(), 0); let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) .expect("history must be non-empty"); @@ -1650,9 +1634,9 @@ fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { Ok(()) } -/// A server that will not prove a transaction at a height says our two chains disagree there. It -/// does not say the transaction is absent from our block, and below the reorg window nothing -/// rewrites that height, so the tip never moves and no tip notification is coming. +/// A server erroring a merkle request says nothing about the block itself — the headers agree +/// throughout — so no reorg is coming to explain it, and below the reorg window nothing about +/// the chain ever changes to trigger a refetch either. The tip stays put. /// /// The script notification is the only thing that comes back, and it can only revive a job that /// still exists — [`ConfirmationJob`] is built in `on_new_tip` and nowhere else. Dropping the job @@ -1665,11 +1649,8 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); let ours = block_with_tx(&header_1, txid, 200, 0); - // Their block at that height does not hold the tx, so they will not prove it there. - let theirs = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); - assert_ne!(ours.merkle_root, theirs.merkle_root); - let mut chain = vec![genesis, header_1, theirs]; + let mut chain = vec![genesis, header_1, ours]; for height in 3..=30u32 { let prev = *chain.last().expect("non-empty"); chain.push(block_with_root( @@ -1680,26 +1661,9 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( )); } - // As in `a_proof_for_another_block_is_not_a_verdict_on_ours`: our block at height 2 is already - // in the cache and the chain, so `GetHeader` does not catch the disagreement first, and the - // agreeing tip keeps any chain job from rewriting that height. - let mut cache = Cache::default(); - cache.headers.insert(ours.block_hash(), ours); - let cp = CheckPoint::new(BlockId { - height: 0, - hash: genesis.block_hash(), - }) - .insert(BlockId { - height: 2, - hash: ours.block_hash(), - }) - .insert(BlockId { - height: 30, - hash: chain[30].block_hash(), - }); - let mut state = new_state_with_cp(cache, descriptor, cp); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); - let mut server = Server { + let server = Server { headers: chain, txs: vec![(tx, 2)], merkle_proof: (Vec::new(), 0), @@ -1722,13 +1686,17 @@ fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( state.poll(&mut queue, resp)?; } assert!( - state.cache().tx_cache.anchors.is_empty(), + state.cache().anchors.is_empty(), "an error must not anchor anything" ); + assert_eq!( + state.chain().tip_height(), + Some(30), + "the tip must have synced past the reorg window despite the error" + ); - // The server comes back to our block at that height. The tip is untouched, so this - // notification is the whole of the recovery. - server.headers[2] = ours; + // The tip is untouched and 28 blocks above height 2 — well past the reorg window a tip + // movement would rewrite — so this notification is the whole of the recovery. let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) .expect("history must be non-empty"); state.poll( @@ -1766,7 +1734,7 @@ fn anchor_survives_a_pass_that_happens_after_it_resolved() -> anyhow::Result<()> let (genesis, header_1) = base_headers(); let header_2 = block_with_tx(&header_1, txid, 200, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1, header_2], @@ -1811,7 +1779,7 @@ fn a_tip_that_moves_while_headers_are_in_flight_is_not_lost() -> anyhow::Result< let (chain_a, chain_b) = (build(0), build(1)); assert_ne!(chain_a[3].block_hash(), chain_b[3].block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: chain_a.clone(), @@ -1884,7 +1852,7 @@ fn headers_for_a_chain_we_were_not_told_about_are_not_adopted() -> anyhow::Resul let b3 = block_with_root(&h2, TxMerkleNode::all_zeros(), 300, 1); assert_ne!(a3.block_hash(), b3.block_hash()); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, h1, h2], @@ -1947,7 +1915,7 @@ fn a_reorg_reanchors_every_script_not_just_the_last_to_notify() -> anyhow::Resul let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1, header_2, header_3], @@ -1958,7 +1926,7 @@ fn a_reorg_reanchors_every_script_not_just_the_last_to_notify() -> anyhow::Resul let anchors_of = |updates: &[Update<&'static str>]| { updates .iter() - .flat_map(|u| u.tx_update.anchors.iter().copied()) + .flat_map(|u| u.tx_update.anchors.iter().cloned()) .collect::>() }; @@ -2026,7 +1994,7 @@ fn a_history_that_cannot_match_the_job_is_not_re_asked() -> anyhow::Result<()> { let txid = tx.compute_txid(); let (genesis, header_1) = base_headers(); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); // The server's chain is one block long, so its history for this script is empty — it will // never answer with the status the notification below carries. @@ -2068,6 +2036,74 @@ fn a_history_that_cannot_match_the_job_is_not_re_asked() -> anyhow::Result<()> { Ok(()) } +/// A history can name a confirmation height above the tip the server has announced: electrs +/// notifies the script hash before the header. That anchor has to *wait* for the tip to catch +/// up. Treating it as history the chain has not backfilled yet sends the job back to plan a run +/// it can never fetch, and the state machine spins. +#[test] +fn an_anchor_above_the_announced_tip_does_not_spin() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut state = new_state(Cache::default(), descriptor); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.start(&mut queue); + drain_requests(&mut state, &mut queue, &server); + + // The server now has the block, so its history reports the transaction at height 2 — but + // only the script hash is notified, which is the order electrs really uses. + server.headers.push(header_2); + let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history is not empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + drain_requests(&mut state, &mut queue, &server); + + assert_eq!( + state.chain().tip_height(), + Some(1), + "no tip was announced, so the chain must not have moved" + ); + assert!( + state.cache().anchors.is_empty(), + "and nothing may be anchored at a height the chain has not reached" + ); + + // The tip catches up, and only now can the anchor resolve. + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_2), "height": 2 }], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "the anchor must be delivered once the tip catches up" + ); + Ok(()) +} + /// The confirmation job must not wait on transactions it never reads. /// /// It works from the heights a history names, so once every script has its history it already @@ -2090,7 +2126,7 @@ fn confirmation_job_runs_ahead_but_the_update_waits_for_the_scripts() -> anyhow: let header_2 = block_with_tx(&header_1, txid, 200, 0); // Deliberately not seeded with the transaction, so the spk job has to ask for it. - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1], @@ -2231,7 +2267,7 @@ fn a_transaction_that_is_not_the_one_asked_for_is_rejected() -> anyhow::Result<( assert_ne!(impostor.compute_txid(), tx.compute_txid()); let (genesis, header_1) = base_headers(); - let mut state = new_state(Cache::default(), descriptor, genesis); + let mut state = new_state(Cache::default(), descriptor); let mut queue = ReqQueue::new(); let server = Server { headers: vec![genesis, header_1], @@ -2267,3 +2303,68 @@ fn a_transaction_that_is_not_the_one_asked_for_is_rejected() -> anyhow::Result<( ); Ok(()) } + +/// A server announcing a tip far below the one we have already verified cannot be synced from: +/// no run is ever planned back that far, so the job would find nothing to do, apply nothing, +/// and hand back the chain we already had as though that server had just confirmed it — then +/// go on asking it for proofs at heights it does not have. +/// +/// This is what an endpoint load-balancing across nodes at different heights looks like, or a +/// node that was rolled back under us. +#[test] +fn a_tip_far_below_the_verified_one_stops_the_connection() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut chain = vec![genesis, header_1, header_2]; + for height in 3..=30u32 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + 0, + )); + } + + let mut state = new_state(Cache::default(), descriptor); + let mut queue = ReqQueue::new(); + let server = Server { + headers: chain.clone(), + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.start(&mut queue); + drain_requests(&mut state, &mut queue, &server); + assert_eq!( + state.chain().tip_height(), + Some(30), + "the chain must be verified up to the tip first" + ); + + // The server now claims a tip 25 blocks below the one we hold — past the window any run + // reaches back to. + let err = state + .poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&chain[5]), "height": 5 }], + })), + ) + .expect_err("a tip that far below the verified one must not be accepted"); + let msg = format!("{err:#}"); + assert!(msg.contains("below the verified tip"), "{msg}"); + + assert_eq!( + state.chain().tip_height(), + Some(30), + "and the verified chain must be left as it was" + ); + Ok(()) +} diff --git a/trusted-headers-gen/Cargo.toml b/trusted-headers-gen/Cargo.toml new file mode 100644 index 0000000..4e2bb01 --- /dev/null +++ b/trusted-headers-gen/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "trusted-headers-gen" +version = "0.1.0" +description = "Fetches a trusted header from Bitcoin Core and writes it out as reviewable Rust source" +license = "MIT OR Apache-2.0" +edition = "2021" +rust-version = "1.85" +repository = "https://github.com/evanlinjin/experiments" + +[dependencies] +anyhow = "1" +bdk_core = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } +corepc-client = { version = "0.13", features = ["client-sync"] } +serde_json = "1" + +[dev-dependencies] +bdk_testenv = { git = "https://github.com/bitcoindevkit/bdk.git", rev = "456f9b7bbf510eefdf3e7a164a5d6bc4cd668adb" } diff --git a/trusted-headers-gen/src/main.rs b/trusted-headers-gen/src/main.rs new file mode 100644 index 0000000..f495cfc --- /dev/null +++ b/trusted-headers-gen/src/main.rs @@ -0,0 +1,571 @@ +//! Fetches a header from a Bitcoin Core node and writes it to `./trusted_headers.rs` (or +//! wherever `--out` says), for whichever network that node is running. +//! +//! Bitcoin Core only ever serves one network per RPC endpoint, so this is meant to be run once +//! per network you want a trusted header for, each time pointed at a node on that network. Every +//! run only touches its own network's section of the output file, adding a new height or +//! replacing an existing one — the other networks' sections are left exactly as they were. +//! +//! What comes out is a Rust module to vendor into whichever crate needs it, exposing a +//! `..._TRUSTED_HEADERS` per network plus a `trusted_headers()` map over them. Nothing ships +//! this data for you: trusting a header means asserting it is canonical, so it belongs in the +//! tree of whoever checked it, having been reviewed as the diff it is. That is also why the +//! output lands in the current directory by default and placing it takes an explicit `--out`. +//! +//! ```text +//! cargo run -p trusted-headers-gen -- \ +//! --url http://127.0.0.1:8332 \ +//! --cookie ~/.bitcoin/.cookie \ +//! --network bitcoin \ +//! [--height 903168] \ +//! [--out trusted_headers.rs] +//! ``` +//! +//! `--height` defaults to the *second* highest difficulty-adjustment boundary (a multiple of +//! 2016) at or below the node's tip — a boundary, because a header-verifying client needs its +//! highest trusted header on one to recompute every retarget above it, and the second one +//! because the highest can sit right at the tip. A lower, explicit `--height` is for backfill: +//! a trusted block below where syncing starts needs no such alignment. + +use std::{ + collections::BTreeMap, + fmt::Write as _, + fs, + path::{Path, PathBuf}, + str::FromStr, +}; + +use bdk_core::bitcoin::{ + block::Header, + consensus::encode::{deserialize_hex, serialize_hex}, + constants::genesis_block, + params::Params, + BlockHash, Network, +}; +use corepc_client::client_sync::{v17::Client, Auth}; + +/// Per-network state: trusted heights and the raw header hex at each. Rebuilt from the existing +/// output file (if any) on every run, so a run for one network cannot lose another's entries. +type State = BTreeMap>; + +const ALL_NETWORKS: [Network; 5] = [ + Network::Bitcoin, + Network::Testnet, + Network::Testnet4, + Network::Signet, + Network::Regtest, +]; + +/// The prefix this network's generated consts carry, e.g. `Network::Bitcoin` -> `MAINNET`. +fn const_prefix(network: Network) -> &'static str { + match network { + Network::Bitcoin => "MAINNET", + Network::Testnet => "TESTNET", + Network::Testnet4 => "TESTNET4", + Network::Signet => "SIGNET", + Network::Regtest => "REGTEST", + } +} + +/// What `getblockchaininfo` calls this network, as defined by BIP70. +fn chain_name(network: Network) -> &'static str { + match network { + Network::Bitcoin => "main", + Network::Testnet => "test", + Network::Testnet4 => "testnet4", + Network::Signet => "signet", + Network::Regtest => "regtest", + } +} + +struct Args { + url: String, + cookie: Option, + user: Option, + pass: Option, + network: Network, + height: Option, + out: PathBuf, +} + +impl Args { + fn parse() -> anyhow::Result { + let mut url = None; + let mut cookie = None; + let mut user = None; + let mut pass = None; + let mut network = None; + let mut height = None; + let mut out = None; + + let mut args = std::env::args().skip(1); + while let Some(flag) = args.next() { + let mut value = || { + args.next() + .ok_or_else(|| anyhow::anyhow!("{flag} needs a value")) + }; + match flag.as_str() { + "--url" => url = Some(value()?), + "--cookie" => cookie = Some(PathBuf::from(value()?)), + "--user" => user = Some(value()?), + "--pass" => pass = Some(value()?), + "--network" => { + network = Some(match value()?.as_str() { + "bitcoin" | "mainnet" => Network::Bitcoin, + "testnet" | "testnet3" => Network::Testnet, + "testnet4" => Network::Testnet4, + "signet" => Network::Signet, + "regtest" => Network::Regtest, + other => anyhow::bail!("unknown --network {other}"), + }) + } + "--height" => height = Some(value()?.parse()?), + "--out" => out = Some(PathBuf::from(value()?)), + other => anyhow::bail!("unknown flag {other}"), + } + } + + Ok(Self { + url: url.ok_or_else(|| anyhow::anyhow!("--url is required"))?, + cookie, + user, + pass, + network: network.ok_or_else(|| anyhow::anyhow!("--network is required"))?, + height, + out: out.unwrap_or_else(|| PathBuf::from("trusted_headers.rs")), + }) + } +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse()?; + + let auth = match (&args.cookie, &args.user, &args.pass) { + (Some(cookie), _, _) => Auth::CookieFile(cookie.clone()), + (None, Some(user), Some(pass)) => Auth::UserPass(user.clone(), pass.clone()), + _ => Auth::None, + }; + let client = if matches!(auth, Auth::None) { + Client::new(&args.url) + } else { + Client::new_with_auth(&args.url, auth).map_err(|e| anyhow::anyhow!(e.to_string()))? + }; + + let (height, hash, header_hex) = fetch_trusted_header(&client, args.network, args.height)?; + + let mut state = parse_existing(&args.out); + let section = state.entry(args.network).or_default(); + section.insert(height, header_hex); + section.insert(0, genesis_hex(args.network)); + fs::write(&args.out, render_file(&state))?; + let _ = std::process::Command::new("rustfmt") + .arg(&args.out) + .status(); + + println!( + "{:?}: trusted header at height {height} ({hash}) written to {}", + args.network, + args.out.display(), + ); + Ok(()) +} + +/// This network's genesis header, hex-encoded. +/// +/// Every section carries one, so the file says which chain each of its headers came from. +/// `HeaderChain::new` refuses a trusted set that has no genesis, and checks the one it has +/// against the params it was handed — without that, a set built for one network can be handed to +/// another's params and pass every later check, because the headers themselves are real. +/// +/// Derived rather than fetched: it is a constant of the network, and the node has already been +/// held to that network by the time we get here. +fn genesis_hex(network: Network) -> String { + serialize_hex(&genesis_block(Params::from(network)).header) +} + +/// Blocks between difficulty adjustments. +const RETARGET_INTERVAL: u32 = 2016; + +/// The height to trust when the caller does not name one: the *second* highest +/// difficulty-adjustment boundary at or below `tip`. +/// +/// A boundary, because the highest trusted block has to sit on one for every retarget above it +/// to be recomputable rather than taken on faith. The second one rather than the highest, +/// because the highest can be the tip itself — and a block a handful of confirmations deep is +/// no basis for trust. Stepping back a whole retarget period puts the anchor between 2016 and +/// 4032 blocks behind the tip, deep enough that reorging past it is not a live concern. +/// +/// Chains too short for that (a fresh regtest, say) fall back to genesis. +fn default_trusted_height(tip: u32) -> u32 { + (tip - tip % RETARGET_INTERVAL).saturating_sub(RETARGET_INTERVAL) +} + +/// Ask the node for the header to trust, returning its height, hash and raw hex. +/// +/// `height` defaults to [`default_trusted_height`]. The node is held to `network` first: a +/// header filed under the wrong network is a poisoned trust anchor, and the node is the one +/// thing that actually knows which chain it is serving. The header is then decoded and checked +/// to hash to the block the node named, so nothing further downstream has to. +fn fetch_trusted_header( + client: &Client, + network: Network, + height: Option, +) -> anyhow::Result<(u32, BlockHash, String)> { + // Read `chain` out of the raw response rather than through a typed one: the shape of the + // rest of `getblockchaininfo` has changed across Core releases, and `chain` is the only + // field here that matters. + let info: serde_json::Value = client + .call("getblockchaininfo", &[]) + .map_err(|e| anyhow::anyhow!("getblockchaininfo: {e}"))?; + let chain = info + .get("chain") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| anyhow::anyhow!("getblockchaininfo gave no `chain`"))?; + anyhow::ensure!( + chain == chain_name(network), + "node is on {chain}, but --network says {network:?} ({})", + chain_name(network), + ); + + let tip = client + .get_block_count() + .map_err(|e| anyhow::anyhow!("getblockcount: {e}"))? + .0 as u32; + let height = height.unwrap_or_else(|| default_trusted_height(tip)); + anyhow::ensure!( + height <= tip, + "height {height} is above the node's tip {tip}" + ); + + let hash = client + .get_block_hash(height as u64) + .map_err(|e| anyhow::anyhow!("getblockhash({height}): {e}"))? + .0; + let hash = + BlockHash::from_str(&hash).map_err(|e| anyhow::anyhow!("server gave a bad hash: {e}"))?; + let header_hex = client + .get_block_header(&hash) + .map_err(|e| anyhow::anyhow!("getblockheader({hash}): {e}"))? + .0; + // Checked once here, so the generated file never needs to check it again. + let header: Header = deserialize_hex(&header_hex)?; + anyhow::ensure!( + header.block_hash() == hash, + "server gave a header for a different block than the hash it named" + ); + Ok((height, hash, header_hex)) +} + +/// Read back whatever `(height, hex)` pairs the file already has, per network — empty if the +/// file does not exist yet, or has nothing for a given network. +fn parse_existing(path: &Path) -> State { + let mut state = State::new(); + let Ok(text) = fs::read_to_string(path) else { + return state; + }; + for network in ALL_NETWORKS { + let prefix = const_prefix(network); + let Some(body) = section_body(&text, prefix) else { + continue; + }; + let mut heights = BTreeMap::new(); + for line in body.lines() { + if let Some((height, hex)) = parse_entry_line(line) { + heights.insert(height, hex); + } + } + if !heights.is_empty() { + state.insert(network, heights); + } + } + state +} + +fn section_body<'a>(text: &'a str, prefix: &str) -> Option<&'a str> { + let begin = format!("// --- BEGIN {prefix} ---"); + let end = format!("// --- END {prefix} ---"); + text.split(&begin).nth(1)?.split(&end).next() +} + +/// Parse a rendered `(height, "hex"),` entry line — the only shape of line this tool ever +/// writes inside a section body. +fn parse_entry_line(line: &str) -> Option<(u32, String)> { + let rest = line.trim().strip_prefix('(')?; + let (height, rest) = rest.split_once(',')?; + let height = height.trim().parse().ok()?; + let hex = rest.trim().strip_prefix('"')?; + let hex = hex.split('"').next()?; + Some((height, hex.to_string())) +} + +/// Render the whole module. `state` is never empty — a run always has a header to add — so +/// every import and helper written here is used by at least one section below. +fn render_file(state: &State) -> String { + let mut out = String::new(); + let w = &mut out; + writeln!( + w, + "// @generated by `cargo run -p trusted-headers-gen`.\n\ + // Do not hand-edit — rerun the generator instead; each run only touches the network\n\ + // it points at." + ) + .unwrap(); + writeln!(w).unwrap(); + writeln!(w, "use std::{{collections::BTreeMap, sync::LazyLock}};").unwrap(); + writeln!(w).unwrap(); + writeln!( + w, + "use bdk_core::bitcoin::{{block::Header, consensus::encode::deserialize_hex, Network}};" + ) + .unwrap(); + writeln!(w).unwrap(); + writeln!( + w, + "/// Deserialize each `(height, hex)` pair — checked valid when this file was generated." + ) + .unwrap(); + writeln!( + w, + "fn parse(raw: [(u32, &str); N]) -> [(u32, Header); N] {{" + ) + .unwrap(); + writeln!( + w, + " raw.map(|(height, hex)| (height, deserialize_hex(hex).expect(\"checked at generation time\")))" + ) + .unwrap(); + writeln!(w, "}}").unwrap(); + writeln!(w).unwrap(); + + for network in ALL_NETWORKS { + let Some(headers) = state.get(&network) else { + continue; + }; + let prefix = const_prefix(network); + let n = headers.len(); + writeln!(w, "// --- BEGIN {prefix} ---").unwrap(); + writeln!( + w, + "const {prefix}_TRUSTED_HEADERS_HEX: [(u32, &str); {n}] = [" + ) + .unwrap(); + for (height, hex) in headers { + writeln!(w, " ({height}, \"{hex}\"),").unwrap(); + } + writeln!(w, "];").unwrap(); + writeln!(w).unwrap(); + writeln!(w, "/// Trusted headers for `Network::{network:?}`.").unwrap(); + writeln!( + w, + "pub static {prefix}_TRUSTED_HEADERS: LazyLock<[(u32, Header); {n}]> =" + ) + .unwrap(); + writeln!( + w, + " LazyLock::new(|| parse({prefix}_TRUSTED_HEADERS_HEX));" + ) + .unwrap(); + writeln!(w, "// --- END {prefix} ---").unwrap(); + writeln!(w).unwrap(); + } + + writeln!( + w, + "/// Every network there are trusted headers here for, keyed by [`Network`]." + ) + .unwrap(); + writeln!( + w, + "pub fn trusted_headers() -> BTreeMap {{" + ) + .unwrap(); + writeln!(w, " let mut map = BTreeMap::new();").unwrap(); + for network in ALL_NETWORKS { + if !state.contains_key(&network) { + continue; + } + let prefix = const_prefix(network); + writeln!( + w, + " map.insert(Network::{network:?}, {prefix}_TRUSTED_HEADERS.as_slice());" + ) + .unwrap(); + } + writeln!(w, " map").unwrap(); + writeln!(w, "}}").unwrap(); + + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_state() -> State { + let mut state = State::new(); + state.insert( + Network::Bitcoin, + BTreeMap::from([(902016, "aa".repeat(80))]), + ); + state.insert( + Network::Signet, + BTreeMap::from([(0, "bb".repeat(80)), (2016, "cc".repeat(80))]), + ); + state + } + + /// The default height must always land on a retarget boundary, and always a whole period + /// behind the tip — never the boundary the tip itself may be sitting on. + #[test] + fn default_height_steps_back_a_whole_retarget_period() { + // 2016 * 450 == 907_200, so that is a boundary and 905_184 is the one below it. + assert_eq!( + default_trusted_height(907_200), + 905_184, + "tip on a boundary" + ); + assert_eq!(default_trusted_height(907_201), 905_184, "just past one"); + assert_eq!(default_trusted_height(907_199), 903_168, "just before one"); + + for tip in [2016_u32, 4032, 100_000, 907_201, 1_000_000] { + let height = default_trusted_height(tip); + assert_eq!(height % RETARGET_INTERVAL, 0, "{height} is not a boundary"); + let depth = tip - height; + assert!( + (RETARGET_INTERVAL..=RETARGET_INTERVAL * 2).contains(&depth), + "tip {tip} put the anchor {depth} blocks deep", + ); + } + + // Chains with no such boundary behind them fall back to genesis. + for tip in [0, 1, 5, 2015] { + assert_eq!(default_trusted_height(tip), 0, "short chain at tip {tip}"); + } + } + + /// The RPC path itself, against a real `bitcoind`: what the tool fetches has to be the + /// header that node actually has at that height, and it has to survive the trip out into + /// generated source and back. + /// + /// Needs a `bitcoind` executable — `BITCOIND_EXE`, a downloaded one, or one on `PATH`. + /// Skipped when there is none, since that is a missing environment rather than a broken + /// tool. + #[test] + fn fetches_a_real_header_from_bitcoin_core() -> anyhow::Result<()> { + let Ok(exe) = bdk_testenv::bitcoind::exe_path() else { + eprintln!("skipping: no bitcoind executable found"); + return Ok(()); + }; + let node = bdk_testenv::bitcoind::BitcoinD::new(exe)?; + let address = node.client.new_address()?; + node.client.generate_to_address(5, &address)?; + + let client = Client::new_with_auth( + &node.rpc_url(), + Auth::CookieFile(node.params.cookie_file.clone()), + ) + .map_err(|e| anyhow::anyhow!(e.to_string()))?; + + // An explicit height, held to the hash the node itself reports for it. + let (height, hash, hex) = fetch_trusted_header(&client, Network::Regtest, Some(3))?; + assert_eq!(height, 3); + assert_eq!(hash, node.client.get_block_hash(3)?.block_hash()?); + assert_eq!(hex.len(), 160, "a header is 80 bytes, so 160 hex chars"); + + // No height given: this chain is far too short to have a whole retarget period behind + // a boundary, so it falls back to genesis. + let (default_height, genesis, _) = fetch_trusted_header(&client, Network::Regtest, None)?; + assert_eq!(default_height, 0); + assert_eq!(genesis, node.client.get_block_hash(0)?.block_hash()?); + + // A height the node cannot answer for is refused, not quietly turned into something. + assert!(fetch_trusted_header(&client, Network::Regtest, Some(500)).is_err()); + + // A regtest node cannot be talked into supplying a header filed under mainnet: the + // node knows which chain it serves, and mislabelling one poisons the trust anchor. + let err = fetch_trusted_header(&client, Network::Bitcoin, Some(3)) + .expect_err("a regtest node must not answer for mainnet") + .to_string(); + assert!(err.contains("node is on regtest"), "{err}"); + + // What came back renders into source that parses back to the very same hex. + let mut state = State::new(); + state + .entry(Network::Regtest) + .or_default() + .insert(height, hex.clone()); + let file = tempfile(); + fs::write(&file, render_file(&state))?; + assert_eq!(parse_existing(&file)[&Network::Regtest][&height], hex); + let _ = fs::remove_file(&file); + Ok(()) + } + + #[test] + fn render_then_parse_round_trips() { + let state = sample_state(); + let rendered = render_file(&state); + let file = tempfile(); + fs::write(&file, &rendered).unwrap(); + assert_eq!(parse_existing(&file), state); + let _ = fs::remove_file(&file); + } + + /// Regenerating for one network must not disturb another's entries, or a run for testnet + /// would wipe out mainnet's trusted header. + #[test] + fn updating_one_network_preserves_the_others() { + let mut state = sample_state(); + let rendered = render_file(&state); + let file = tempfile(); + fs::write(&file, &rendered).unwrap(); + + let mut reloaded = parse_existing(&file); + reloaded + .entry(Network::Testnet) + .or_default() + .insert(4032, "dd".repeat(80)); + fs::write(&file, render_file(&reloaded)).unwrap(); + + state + .entry(Network::Testnet) + .or_default() + .insert(4032, "dd".repeat(80)); + assert_eq!(parse_existing(&file), state); + let _ = fs::remove_file(&file); + } + + /// A fresh height for a network already in the file adds to it; the same height replaces. + #[test] + fn same_height_replaces_new_height_adds() { + let state = sample_state(); + let file = tempfile(); + fs::write(&file, render_file(&state)).unwrap(); + + let mut reloaded = parse_existing(&file); + reloaded + .get_mut(&Network::Bitcoin) + .unwrap() + .insert(902016, "ee".repeat(80)); // replace + reloaded + .get_mut(&Network::Bitcoin) + .unwrap() + .insert(904032, "ff".repeat(80)); // add + fs::write(&file, render_file(&reloaded)).unwrap(); + + let mainnet = &parse_existing(&file)[&Network::Bitcoin]; + assert_eq!(mainnet.get(&902016), Some(&"ee".repeat(80))); + assert_eq!(mainnet.get(&904032), Some(&"ff".repeat(80))); + assert_eq!(mainnet.len(), 2); + + let _ = fs::remove_file(&file); + } + + fn tempfile() -> PathBuf { + std::env::temp_dir().join(format!( + "trusted_headers_gen_test_{:?}_{}", + std::thread::current().id(), + std::process::id(), + )) + } +}