diff --git a/Cargo.lock b/Cargo.lock index a1a3d96..47fb2cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1146,11 +1146,71 @@ dependencies = [ "chrono", "clap", "config", - "dolos-cardano", - "dolos-core", - "dolos-fjall", - "dolos-redb3", - "dolos-snapshot", + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-fjall 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-redb3 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-snapshot 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "flate2", + "futures-core", + "futures-util", + "gasket", + "hex", + "http", + "hyper", + "indicatif", + "itertools 0.14.0", + "lazy_static", + "log", + "miette", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "pallas", + "paste", + "protoc-wkt", + "rayon", + "redb-extras", + "reqwest 0.12.28", + "rustls", + "serde", + "serde_json", + "tar", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-util", + "tower 0.4.13", + "tower-http", + "tracing", + "tracing-log", + "tracing-opentelemetry", + "tracing-subscriber", + "trait-variant", + "vergen-gitcl", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "dolos" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.22.1", + "bech32 0.11.1", + "bincode 1.3.3", + "bytes", + "chrono", + "clap", + "config", + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-fjall 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-redb3 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-snapshot 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "flate2", "futures-core", "futures-util", @@ -1201,7 +1261,32 @@ dependencies = [ "async-trait", "chrono", "crc", - "dolos-core", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "futures-core", + "hex", + "itertools 0.14.0", + "num-bigint", + "num-rational", + "pallas", + "paste", + "rayon", + "self_cell", + "serde", + "serde_json", + "thiserror 2.0.20", + "tracing", +] + +[[package]] +name = "dolos-cardano" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "crc", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "futures-core", "hex", "itertools 0.14.0", @@ -1241,14 +1326,54 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "dolos-core" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "futures-core", + "futures-util", + "hex", + "itertools 0.14.0", + "opentelemetry", + "pallas", + "rayon", + "regex", + "serde", + "serde_with 3.23.0", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tracing", + "tracing-opentelemetry", + "trait-variant", + "xxhash-rust", +] + [[package]] name = "dolos-fjall" version = "2.0.0-alpha.0" source = "git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30#1ae4e91c18a9e1456a3612af402d7b9b97546d30" dependencies = [ "bincode 1.3.3", - "dolos-core", - "dolos-flatfiles", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-flatfiles 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "fjall", + "pallas", + "tempfile", + "thiserror 2.0.20", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "dolos-fjall" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "bincode 1.3.3", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-flatfiles 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "fjall", "pallas", "tempfile", @@ -1267,12 +1392,22 @@ dependencies = [ "zstd", ] +[[package]] +name = "dolos-flatfiles" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "rayon", + "tempfile", + "zstd", +] + [[package]] name = "dolos-mithril" version = "2.0.0-alpha.0" -source = "git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30#1ae4e91c18a9e1456a3612af402d7b9b97546d30" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" dependencies = [ - "dolos-core", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "mithril-client", "tracing", ] @@ -1283,7 +1418,30 @@ version = "2.0.0-alpha.0" source = "git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30#1ae4e91c18a9e1456a3612af402d7b9b97546d30" dependencies = [ "bincode 1.3.3", - "dolos-core", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "futures-core", + "futures-util", + "hex", + "itertools 0.14.0", + "pallas", + "redb", + "serde", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tokio-util", + "tower-http", + "tracing", +] + +[[package]] +name = "dolos-redb3" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "bincode 1.3.3", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "futures-core", "futures-util", "hex", @@ -1305,8 +1463,25 @@ name = "dolos-snapshot" version = "2.0.0-alpha.0" source = "git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30#1ae4e91c18a9e1456a3612af402d7b9b97546d30" dependencies = [ - "dolos-cardano", - "dolos-core", + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "hex", + "minicbor 0.26.5", + "serde", + "serde_json", + "stelae", + "stelae-driver", + "thiserror 2.0.20", + "tracing", +] + +[[package]] +name = "dolos-snapshot" +version = "2.0.0-alpha.0" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" +dependencies = [ + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "hex", "minicbor 0.26.5", "serde", @@ -1320,13 +1495,13 @@ dependencies = [ [[package]] name = "dolos-testing" version = "2.0.0-alpha.0" -source = "git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30#1ae4e91c18a9e1456a3612af402d7b9b97546d30" +source = "git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5#2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" dependencies = [ "bech32 0.9.1", - "dolos-cardano", - "dolos-core", - "dolos-fjall", - "dolos-redb3", + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-fjall 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-redb3 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "futures-core", "futures-util", "hex", @@ -4641,12 +4816,14 @@ dependencies = [ name = "stelae-cardano" version = "0.2.0" dependencies = [ - "dolos", - "dolos-cardano", - "dolos-core", - "dolos-flatfiles", + "dolos 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-cardano 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-core 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", + "dolos-flatfiles 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "dolos-mithril", - "dolos-snapshot", + "dolos-snapshot 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=1ae4e91c18a9e1456a3612af402d7b9b97546d30)", + "dolos-snapshot 2.0.0-alpha.0 (git+https://github.com/txpipe/dolos?rev=2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5)", "dolos-testing", "hex", "itertools 0.14.0", diff --git a/Cargo.toml b/Cargo.toml index 0712eee..c234335 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,11 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/txpipe/stelae" -# Dolos v2.0.0-alpha.0 pins the released v0.2.0 Stelae crates. The publisher -# is built in this workspace, so patch both names back to these paths: this is -# what makes the profile's trait and registry types identical to the host's, -# rather than two same-version crates from different sources. +# The accepted Dolos publisher-retirement revision pins the released v0.2.0 +# Stelae crates. The publisher is built in this workspace, so patch both names +# back to these paths: this is what makes the profile's trait and registry +# types identical to the host's, rather than two same-version crates from +# different sources. [patch."https://github.com/txpipe/stelae"] stelae = { path = "stelae" } stelae-driver = { path = "stelae-driver" } diff --git a/docs/publisher-packaging.md b/docs/publisher-packaging.md index ebcbca7..25ea42b 100644 --- a/docs/publisher-packaging.md +++ b/docs/publisher-packaging.md @@ -45,10 +45,30 @@ identify the released image and source revision. Native binaries are only intermediate workflow artifacts; there are no standalone binary releases or custom provenance files. +## Candidate identity + +A candidate is the tuple of the full Stelae commit, committed `Cargo.lock`, +accepted Dolos revision +`2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5`, chart source at that same Stelae +commit, and reviewed values/configuration. Record all of them at handoff. Before +publication the image digest is intentionally unknown; use the commit as the +candidate identity, then add the registry digest and attestation after the +explicit tag-driven publication. Never substitute a moving image tag for the +missing digest. + +The publisher image and its Dolos profile/protocol dependencies do not need an +artificial release cadence. The immutable git pin and lockfile make the exact +combination reproducible, so a publisher candidate can be tested without a new +Dolos release, and a Dolos or Stelae protocol release need not publish a new +publisher image when its locked inputs did not change. This repository's one +workspace tag remains a source identity for the Stelae crates and application; +publication of protocol crates, image artifacts and deployment are still +distinct, explicit decisions. + ## Release -1. Verify the candidate commit passed the workspace, parity, registry, image - and chart checks. +1. Verify the candidate tuple above passed the workspace, parity, registry, + image and chart checks. 2. Prepare the version/changelog with the repository's `cargo release` flow and review the release commit and signed `v` tag before pushing. 3. Pushing the version tag triggers image publication after both architecture diff --git a/docs/publisher-parity.md b/docs/publisher-parity.md index f787efc..8934cba 100644 --- a/docs/publisher-parity.md +++ b/docs/publisher-parity.md @@ -9,6 +9,9 @@ wire format or the Dolos consumer. - old host: Dolos `1ae4e91c18a9e1456a3612af402d7b9b97546d30` (`dolos snapshot publish` and its headless snapshot facade); - new host base: Stelae `47b38a4`, the merge of publisher-host PR 3; +- final Dolos consumer/facade graph: + `2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5`, the merge of publisher-retirement + PR 1335; - protocol/profile: Stelae 0.2.0 and `io.txpipe.dolos.cardano` v1; - real fixture: `stelae-cardano/tests/fixtures/preview-epoch-0`, identified and explained by its README; @@ -33,10 +36,11 @@ cargo test --locked -p stelae-cardano --test publisher_parity \ -- --ignored --nocapture ``` -The JSON report identifies both hosts and all pins, records pass/fail checks, -and measures replay/publish wall time, process peak RSS, artifact size, -transfer, and scratch use. Directory publication moves zero network bytes and -uses no OCI scratch, so both values are explicitly zero rather than omitted. +The JSON report identifies both hosts, the final Dolos dependency pin and all +fixture/profile pins, records pass/fail checks, and measures replay/publish +wall time, process peak RSS, artifact size, transfer, and scratch use. Directory +publication moves zero network bytes and uses no OCI scratch, so both values +are explicitly zero rather than omitted. The report also records each host's first-push layer and byte transfer counters from separate repositories in a disposable `registry:2`. CI runs the command in the `Real publisher parity` job and uploads `publisher-parity-report`. @@ -95,9 +99,11 @@ exercise the orchestration seams around it. ## Scope and interpretation All repositories are loopback/disposable and no production registry is named. -The old source is required only by immutable git revision, not by a future -Dolos release branch. The real fixture and genesis inputs live here by content -identity, so removal of the old Dolos command does not remove the evidence. +The old source is required only by immutable, test-only git aliases; its package +types remain separate from the final Dolos graph used by `stelae-publisher` and +the consumer restore checks. It needs no future Dolos release branch. The real +fixture and genesis inputs live here by content identity, so removal of the old +Dolos command does not remove the evidence. This gate establishes code-path parity for packaging. Production canary, soak, and mainnet performance remain separate operations and must not be inferred diff --git a/docs/publisher.md b/docs/publisher.md index f462ae5..1dd502b 100644 --- a/docs/publisher.md +++ b/docs/publisher.md @@ -8,13 +8,22 @@ No command starts or shells out to a Dolos executable. ## Source and dependency identity -All imported Dolos crates are pinned to -`1ae4e91c18a9e1456a3612af402d7b9b97546d30`, the merge revision of -[Dolos PR 1334](https://github.com/txpipe/dolos/pull/1334). The root `dolos` -package is imported with default features disabled. The workspace patch for -`https://github.com/txpipe/stelae` resolves the profile's v0.2.0 dependencies -back to the workspace `stelae` and `stelae-driver` packages. This prevents two -same-version protocol type universes. +All Dolos crates in the normal publisher dependency graph are pinned to +`2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5`, the merge revision of +[Dolos PR 1335](https://github.com/txpipe/dolos/pull/1335). This is the +publisher-retirement revision: it retains the headless ledger, storage, +profile, restore and repository facades but contains no publisher command, +backfill loop or publisher policy module. The root `dolos` package is imported +with default features disabled, so its service features are absent from the +publisher graph. + +The workspace patch for `https://github.com/txpipe/stelae` resolves the +profile's v0.2.0 dependencies back to the workspace `stelae` and +`stelae-driver` packages. This prevents two same-version protocol type +universes. The step-4 parity test retains the former Dolos host at immutable +revision `1ae4e91c18a9e1456a3612af402d7b9b97546d30` through explicitly named, +test-only dependency aliases; those aliases are reference evidence and are not +in the publisher's normal dependency graph. The lockfile is committed. These commands verify the intended graph: @@ -119,18 +128,19 @@ The backfill state machine preserves these rules from the accepted Dolos host: - retain `snapshot.state_epochs` and the profile's existing plan, schema, media-type, compression and journal identities. -## Finite duplication window +## Final ownership boundary -Dolos still carries its old `snapshot publish` / `snapshot backfill` command -and backfill module at the pinned revision. This host does not call that -module: `stelae-cardano/src/backfill.rs` owns the running loop. The duplicate is -temporary evidence for parity. Publisher-pipeline step 6 removes the old Dolos -implementation after parity, and step 7 removes the stale dependency pin. +`stelae-cardano/src/backfill.rs` is the only running publisher loop, and +`stelae-cardano/src/publisher.rs` owns the host's repository-standing and +journal policy. It composes Dolos's supported `SnapshotRepository`, +`Publishing` and `SnapshotSource` facades; it does not import a Dolos publisher +or backfill API. The immutable old-host aliases used by the parity test are not +linked into `stelae-publisher`. The image build, release identities, chart cutover and compatible rollback are documented in [publisher-packaging.md](publisher-packaging.md). Building a candidate does not publish, deploy, modify a network source, or claim production -parity; operational cutover and later pin cleanup remain separate steps. +parity; operational cutover remains a separate step. The executable old/new replay, publication, recovery, resource, and local OCI evidence is documented in [publisher-parity.md](publisher-parity.md). diff --git a/stelae-cardano/Cargo.toml b/stelae-cardano/Cargo.toml index b086dad..5c6e954 100644 --- a/stelae-cardano/Cargo.toml +++ b/stelae-cardano/Cargo.toml @@ -10,12 +10,12 @@ repository.workspace = true stelae = { path = "../stelae", version = "=0.2.0", features = ["oci"] } stelae-driver = { path = "../stelae-driver", version = "=0.2.0", features = ["oci"] } -# Accepted publisher-pipeline facade revision: txpipe/dolos PR #1334. -dolos = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30", default-features = false } -dolos-cardano = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } -dolos-core = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } -dolos-mithril = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } -dolos-snapshot = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } +# Accepted publisher-retirement revision: txpipe/dolos PR #1335. +dolos = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5", default-features = false } +dolos-cardano = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } +dolos-core = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } +dolos-mithril = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } +dolos-snapshot = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } itertools = "0.14.0" pallas = { version = "1.1.1", features = ["hardano", "phase2", "unstable"] } @@ -27,8 +27,12 @@ tokio-util = { version = "0.7.11", features = ["rt"] } tracing = "0.1.37" [dev-dependencies] -dolos-flatfiles = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } -dolos-testing = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } +dolos-flatfiles = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } +dolos-testing = { git = "https://github.com/txpipe/dolos", rev = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5" } +# Step-4's immutable comparison host remains test-only after Dolos removes it. +# Aliases keep its types visibly separate from the final consumer graph above. +dolos-old = { package = "dolos", git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30", default-features = false } +dolos-snapshot-old = { package = "dolos-snapshot", git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } hex = "0.4.3" libc = "0.2.186" rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } diff --git a/stelae-cardano/src/lib.rs b/stelae-cardano/src/lib.rs index daae54a..68720b8 100644 --- a/stelae-cardano/src/lib.rs +++ b/stelae-cardano/src/lib.rs @@ -16,8 +16,8 @@ pub use dolos_snapshot::{ restore::Source as RestoreSource, }; -/// The exact Dolos revision accepted by publisher-pipeline step 2. -pub const DOLOS_REVISION: &str = "1ae4e91c18a9e1456a3612af402d7b9b97546d30"; +/// The exact Dolos revision accepted by publisher-pipeline step 6. +pub const DOLOS_REVISION: &str = "2cadf62a2cd9b15ed1c17a9816f976a8646ac2e5"; #[cfg(test)] mod dependency_identity { diff --git a/stelae-cardano/src/publisher.rs b/stelae-cardano/src/publisher.rs index 532f6c8..45071d2 100644 --- a/stelae-cardano/src/publisher.rs +++ b/stelae-cardano/src/publisher.rs @@ -8,7 +8,9 @@ use dolos_snapshot::{ facade::SnapshotSource, node, planning::{self, PlanReport}, - registry::{self, Auth, Preview, Published, Repository, Tuning}, + registry::{ + self, Auth, Preview, Published, Publishing, Repository, SnapshotRepository, Tuning, + }, }; use stelae::progress::Observer; use stelae_driver::Standing; @@ -33,11 +35,7 @@ pub enum Next { } impl Next { - pub fn read( - standing: Standing, - sequence: u64, - require_new: bool, - ) -> Result { + pub fn read(standing: Standing, sequence: u64, require_new: bool) -> Result { match standing { Standing::Empty => Ok(Self::First), Standing::Next { latest } => Ok(Self::After { latest }), @@ -46,12 +44,12 @@ impl Next { "nothing to publish: this repository is at sequence {latest} and this node is at sequence {sequence}" ); if require_new { - Err(dolos_snapshot::Error::NothingToPublish(message)) + Err(PolicyError::NothingToPublish(message)) } else { Ok(Self::Nothing(message)) } } - Standing::Ahead { latest, distance } => Err(dolos_snapshot::Error::PublishWouldGap { + Standing::Ahead { latest, distance } => Err(PolicyError::PublishWouldGap { latest, sequence, distance, @@ -60,9 +58,29 @@ impl Next { } } +/// Publisher-host refusals that do not belong to the Dolos profile facade. +#[derive(Debug, thiserror::Error)] +pub enum PolicyError { + #[error("{0}")] + NothingToPublish(String), + + #[error( + "this repository's latest stele is sequence {latest} and this node is at sequence \ + {sequence}, {distance} sequences ahead: a publish must follow the repository's latest \ + stele, and this one would leave a gap no later stele could close" + )] + PublishWouldGap { + latest: u64, + sequence: u64, + distance: u64, + }, +} + /// An opened repository plus the journal and reuse policy of this publisher. pub struct Publisher { - inner: dolos_snapshot::publisher::Publisher, + repository: SnapshotRepository, + record_path: PathBuf, + rebuild: bool, } impl Publisher { @@ -93,25 +111,21 @@ impl Publisher { rebuild: bool, tuning: Tuning, ) -> Result { - let inner = dolos_snapshot::publisher::Publisher::open_explicit( + let repository = SnapshotRepository::open(repository, insecure, auth, scratch_dir, tuning)?; + + Ok(Self { repository, - insecure, - auth, - scratch_dir, record_path, rebuild, - tuning, - )?; - - Ok(Self { inner }) + }) } pub fn standing(&self, plan: &Plan) -> Result { - self.inner.standing(plan) + self.repository.standing(plan) } pub fn preflight(&self) -> Result<(), dolos_snapshot::Error> { - self.inner.preflight() + self.repository.preflight() } pub fn preview( @@ -119,7 +133,7 @@ impl Publisher { plan: &Plan, source: &dyn SnapshotSource, ) -> Result { - source.preview(&self.inner, plan) + source.preview_repository(self.publishing(), plan) } pub fn publish( @@ -128,7 +142,14 @@ impl Publisher { source: &dyn SnapshotSource, observer: &Observer, ) -> Result { - source.publish(&self.inner, plan, observer) + source.publish_repository(self.publishing(), plan, observer) + } + + fn publishing(&self) -> Publishing<'_> { + self.repository + .publishing() + .recording_in(self.record_path.clone()) + .rebuilding(self.rebuild) } } diff --git a/stelae-cardano/tests/publisher_parity.rs b/stelae-cardano/tests/publisher_parity.rs index e6865b9..63afabb 100644 --- a/stelae-cardano/tests/publisher_parity.rs +++ b/stelae-cardano/tests/publisher_parity.rs @@ -16,14 +16,20 @@ use std::{ time::{Duration, Instant}, }; -use dolos::engine::{BulkReplaySession, ReplayWorkspace}; +use dolos::engine::BulkReplaySession; use dolos_core::{config::RootConfig, Genesis, ReplayProgress}; use dolos_flatfiles::{BlockLocation, FlatFileStore}; -use dolos_snapshot::{ - facade::SnapshotSource, - planning, +use dolos_old::engine::{ + BulkReplaySession as OldBulkReplaySession, ReplayWorkspace as OldReplayWorkspace, +}; +use dolos_snapshot::registry::{ + Auth, Point as RepositoryPoint, Repository, SnapshotRepository, Tuning, +}; +use dolos_snapshot_old::{ + facade::SnapshotSource as OldSnapshotSource, + planning as old_planning, publisher::{Next as OldNext, Publisher as OldPublisher, RepositoryPublish as OldPublish}, - registry::{Auth, Point as RepositoryPoint, Published, Repository, SnapshotRepository, Tuning}, + registry::{Auth as OldAuth, Published as OldPublished, Tuning as OldTuning}, }; use pallas::ledger::traverse::MultiEraBlock; use stelae::progress::Observer; @@ -47,34 +53,58 @@ struct Node { config: RootConfig, } +struct OldNode { + root: tempfile::TempDir, + config: dolos_old::core::config::RootConfig, +} + +impl OldNode { + fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let document = node_config(root.path()); + Self { + root, + config: toml::from_str(&document).unwrap(), + } + } + + fn stele(&self, name: &str) -> std::path::PathBuf { + self.root.path().join(name) + } +} + +fn node_config(root: &Path) -> String { + format!( + r#" + [upstream] + peer_address = "unused.invalid:3001" + + [storage] + version = "v4" + path = {} + + [genesis] + byron_path = "unused" + shelley_path = "unused" + alonzo_path = "unused" + conway_path = "unused" + + [snapshot] + state_epochs = [1] + + [chain] + type = "cardano" + magic = 2 + is_testnet = true + "#, + toml::Value::String(root.join("data").display().to_string()), + ) +} + impl Node { fn new() -> Self { let root = tempfile::tempdir().unwrap(); - let document = format!( - r#" - [upstream] - peer_address = "unused.invalid:3001" - - [storage] - version = "v4" - path = {} - - [genesis] - byron_path = "unused" - shelley_path = "unused" - alonzo_path = "unused" - conway_path = "unused" - - [snapshot] - state_epochs = [1] - - [chain] - type = "cardano" - magic = 2 - is_testnet = true - "#, - toml::Value::String(root.path().join("data").display().to_string()), - ); + let document = node_config(root.path()); Self { root, config: toml::from_str(&document).unwrap(), @@ -102,6 +132,18 @@ fn fixture_genesis() -> Genesis { .unwrap() } +fn old_fixture_genesis() -> dolos_old::core::Genesis { + let root = fixture_path().parent().unwrap().to_owned(); + dolos_old::core::Genesis::from_file_paths( + root.join("byron.json"), + root.join("shelley.json"), + root.join("alonzo.json"), + root.join("conway.json"), + Some(6), + ) + .unwrap() +} + fn read_fixture() -> Vec>> { let path = fixture_path(); for (name, expected) in [ @@ -178,20 +220,34 @@ fn replay(node: &Node, blocks: Vec>>) -> (u64, u128) { (position.slot(), started.elapsed().as_millis()) } -fn old_host_publish(node: &Node, output: &Path) -> (String, u128) { +fn old_replay(node: &OldNode, blocks: Vec>>) -> (u64, u128) { let started = Instant::now(); - let genesis = Arc::new(fixture_genesis()); - let workspace = ReplayWorkspace::open(&node.config, genesis.clone()).unwrap(); + let genesis = Arc::new(old_fixture_genesis()); + let mut replay = OldBulkReplaySession::open(&node.config, genesis, Some(1)).unwrap(); + let dolos_old::core::ReplayProgress::Boundary { position } = + replay.import_blocks(blocks).unwrap() + else { + panic!("real fixture did not stop at the first Preview epoch boundary") + }; + assert!(position.slot() >= PREVIEW_EPOCH_LENGTH); + replay.finish().unwrap(); + (position.slot(), started.elapsed().as_millis()) +} + +fn old_host_publish(node: &OldNode, output: &Path) -> (String, u128) { + let started = Instant::now(); + let genesis = Arc::new(old_fixture_genesis()); + let workspace = OldReplayWorkspace::open(&node.config, genesis.clone()).unwrap(); let result = (|| { let snapshot = workspace.snapshot(); - let retained = planning::retained_epochs(&node.config)?; + let retained = old_planning::retained_epochs(&node.config)?; let plan = snapshot.selected_plan( u64::from(genesis.network_magic()), retained, - Selection::default(), + dolos_snapshot_old::facade::Selection::default(), )?; let inscription = snapshot.publish_directory(output, &plan, &Observer::silent())?; - Ok::<_, dolos_snapshot::Error>(inscription.digest()?.to_string()) + Ok::<_, dolos_snapshot_old::Error>(inscription.digest()?.to_string()) })(); workspace.finish().unwrap(); (result.unwrap(), started.elapsed().as_millis()) @@ -216,20 +272,20 @@ fn new_host_publish(node: &Node, output: &Path) -> (String, u128) { } fn old_host_publish_repository( - node: &Node, + node: &OldNode, repository: &Repository, rebuild: bool, dry_run: bool, require_new: bool, -) -> Result, AnyError> { - let genesis = Arc::new(fixture_genesis()); - let workspace = ReplayWorkspace::open(&node.config, genesis.clone())?; - let result: Result, dolos_snapshot::Error> = (|| { +) -> Result, AnyError> { + let genesis = Arc::new(old_fixture_genesis()); + let workspace = OldReplayWorkspace::open(&node.config, genesis.clone())?; + let result: Result, dolos_snapshot_old::Error> = (|| { let snapshot = workspace.snapshot(); let plan = snapshot.selected_plan( u64::from(genesis.network_magic()), - planning::retained_epochs(&node.config)?, - Selection::default(), + old_planning::retained_epochs(&node.config)?, + dolos_snapshot_old::facade::Selection::default(), )?; let settings = OldPublish { repo: repository, @@ -238,9 +294,9 @@ fn old_host_publish_repository( rebuild, dry_run, require_new, - tuning: Tuning::default(), + tuning: OldTuning::default(), }; - let publisher = OldPublisher::open(&node.config, &settings, Auth::Anonymous)?; + let publisher = OldPublisher::open(&node.config, &settings, OldAuth::Anonymous)?; match OldNext::read(publisher.standing(&plan)?, plan.sequence, require_new)? { OldNext::Nothing(_) => Ok(None), OldNext::First | OldNext::After { .. } => { @@ -493,9 +549,9 @@ fn peak_rss_bytes() -> u64 { #[ignore = "replays 4,958 real Preview blocks; run by the parity CI lane"] fn old_and_new_hosts_replay_publish_and_restore_the_same_real_boundary() { let fixture = read_fixture(); - let old = Node::new(); + let old = OldNode::new(); let new = Node::new(); - let (old_slot, old_replay_ms) = replay(&old, fixture.clone()); + let (old_slot, old_replay_ms) = old_replay(&old, fixture.clone()); let (new_slot, new_replay_ms) = replay(&new, fixture); assert_eq!(old_slot, new_slot); @@ -513,7 +569,7 @@ fn old_and_new_hosts_replay_publish_and_restore_the_same_real_boundary() { let restored = restore_with_unchanged_consumer(&new_dir); let restored_dir = restored.stele("restored-stele"); - let (restored_identity, _) = old_host_publish(&restored, &restored_dir); + let (restored_identity, _) = new_host_publish(&restored, &restored_dir); assert_eq!(new_identity, restored_identity); let registry = LocalRegistry::spawn(); @@ -589,7 +645,7 @@ fn old_and_new_hosts_replay_publish_and_restore_the_same_real_boundary() { let restored_registry = restore_repository_with_unchanged_consumer(new_repository.clone()); let restored_registry_dir = restored_registry.stele("restored-registry-stele"); let (restored_registry_identity, _) = - old_host_publish(&restored_registry, &restored_registry_dir); + new_host_publish(&restored_registry, &restored_registry_dir); assert_eq!(new_registry_identity, restored_registry_identity); let report = serde_json::json!({ @@ -599,6 +655,7 @@ fn old_and_new_hosts_replay_publish_and_restore_the_same_real_boundary() { "old": {"revision": OLD_HOST_REVISION, "kind": "dolos snapshot publish"}, "new": { "revision": option_env!("STELAE_PARITY_NEW_REVISION").unwrap_or("worktree"), + "dolos_revision": stelae_cardano::DOLOS_REVISION, "kind": "stelae-publisher" } },