diff --git a/.cairn/.gitignore b/.cairn/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/.cairn/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/.cairn/README.md b/.cairn/README.md new file mode 100644 index 0000000..540492e --- /dev/null +++ b/.cairn/README.md @@ -0,0 +1,17 @@ +# Boros — Cairn dossier + +This tree is maintained under the [Cairn](https://github.com/txpipe) discipline and its layout is framework-defined: derived files are regenerated, never hand-edited; intent lives in the colocated manifests (`WORKSPACE.md`, `src/*/MODULE.md`); decisions live in [`decisions/`](decisions/). `out/` is machine exhaust and never committed. + +Boros ("tx omnivore") is a Cardano transaction submission service: it accepts raw transactions over gRPC, queues them with priority and chaining semantics in SQLite, fans them out to Ouroboros tx-submission peers, and tracks their fate on-chain (confirmation, retry, rollback) via UtxoRPC chainsync. + +## Reading order + +1. [life-map.md](life-map.md) — where the action is: churn × complexity partition, chartering nominations. **Repo dormant since 2025-05, judged "will resume" (human, 2026-08-23).** +2. [topology.md](topology.md) — module graph, instability table, the one root cycle, confirmed layer labels. Rendered crate graph: [crate-graph.svg](crate-graph.svg). +3. The four flows — the tx lifecycle (`Pending → InFlight → Confirmed | Failed`, defined in `src/storage/mod.rs`) is the spine; each flow drives one leg: + - [flows/submit.md](flows/submit.md) — gRPC → validated → queued (`Pending`). + - [flows/fanout.md](flows/fanout.md) — priority drain → sign → broadcast (`InFlight`/`Failed`). + - [flows/confirm.md](flows/confirm.md) — chainsync → `Confirmed`/retry/rollback + cursor. **Flow-contract nominee.** + - [flows/peer-discovery.md](flows/peer-discovery.md) — relay/peer-sharing pool top-up (relay side mocked — [decision 0003](decisions/0003-real-relay-source-for-peer-discovery.md)). + +Reference material, consulted rather than read in order: [shape.md](shape.md) (census: size, tests, unsafe, dependency weight), [api/](api/) (internal public-item surface — bin-only crate, so this is the operational-comprehension snapshot, not a semver surface), [context-map.md](context-map.md) (generated from manifest relationships), [decisions/](decisions/) (ADRs 0001–0004: gasket fork, dependency pinning, relay mock, `Validated` status). diff --git a/.cairn/api/boros-internal-surface.txt b/.cairn/api/boros-internal-surface.txt new file mode 100644 index 0000000..5ecbdcf --- /dev/null +++ b/.cairn/api/boros-internal-surface.txt @@ -0,0 +1,171 @@ +GENERATED 2026-08-23 by cairn-survey @ fdec4da via `cargo modules structure --bin boros`. +NOTE: internal public-item surface of a bin-only crate — NOT a semver API surface (cargo public-api inapplicable; see shape.md deviation notes). + + +crate boros +├── struct Config: pub(crate) +│ └── fn new: pub +├── mod ledger: pub(crate) +│ ├── mod relay: pub +│ │ ├── struct MockRelayDataAdapter: pub +│ │ │ ├── async fn get_relays: pub(self) +│ │ │ └── fn new: pub +│ │ └── trait RelayDataAdapter: pub +│ └── mod u5c: pub +│ ├── type ChainSyncStream: pub +│ ├── struct Config: pub +│ ├── enum Event: pub +│ ├── type Point: pub +│ ├── trait U5cDataAdapter: pub +│ ├── struct U5cDataAdapterImpl: pub +│ │ ├── async fn fetch_pparams: pub(self) +│ │ ├── async fn fetch_tip: pub(self) +│ │ ├── async fn fetch_utxos: pub(self) +│ │ ├── fn interceptor: pub(self) +│ │ ├── async fn stream: pub(self) +│ │ └── async fn try_new: pub +│ ├── fn map_conway_pparams: pub(self) +│ └── fn map_multi_era_pparams: pub(self) +├── async fn main: pub(crate) +├── mod network: pub(crate) +│ ├── mod mempool: pub +│ │ ├── struct Event: pub +│ │ ├── struct Mempool: pub +│ │ │ ├── fn acknowledge: pub +│ │ │ ├── fn find_inflight: pub +│ │ │ ├── fn new: pub +│ │ │ ├── fn notify: pub +│ │ │ ├── fn receive: pub(self) +│ │ │ └── fn receive_raw: pub +│ │ ├── enum MempoolError: pub +│ │ ├── struct MempoolState: pub(self) +│ │ ├── struct Tx: pub +│ │ ├── type TxHash: pub(self) +│ │ └── enum TxStage: pub +│ ├── mod mock_ouroboros_tx_submit_server: pub +│ │ └── struct MockOuroborosTxSubmitPeerServer: pub +│ │ ├── async fn init: pub +│ │ ├── fn new: pub +│ │ └── async fn start_background_task: pub(self) +│ ├── mod peer: pub +│ │ ├── struct Peer: pub +│ │ │ ├── async fn collect_transactions: pub(self) +│ │ │ ├── async fn discover_peers: pub +│ │ │ ├── async fn init: pub +│ │ │ ├── fn new: pub +│ │ │ ├── async fn propagate_txs: pub(self) +│ │ │ ├── async fn query_peer_sharing_mode: pub +│ │ │ └── fn start_background_task: pub(self) +│ │ └── enum PeerError: pub +│ └── mod peer_manager: pub +│ ├── struct PeerManager: pub +│ │ ├── async fn add_peer: pub +│ │ ├── async fn connected_peers_count: pub +│ │ ├── async fn init: pub +│ │ ├── async fn is_peer_exist: pub +│ │ ├── fn new: pub +│ │ └── async fn pick_peer_rand: pub +│ ├── struct PeerManagerConfig: pub +│ └── enum PeerManagerError: pub +├── mod pipeline: pub(crate) +│ ├── mod ingest: pub +│ │ ├── struct Stage: pub +│ │ │ └── fn new: pub +│ │ └── struct Worker: pub +│ │ ├── async fn bootstrap: pub(self) +│ │ ├── async fn execute: pub(self) +│ │ └── async fn schedule: pub(self) +│ ├── mod monitor: pub +│ │ ├── struct Config: pub +│ │ ├── struct Stage: pub +│ │ │ └── fn new: pub +│ │ └── struct Worker: pub +│ │ ├── async fn bootstrap: pub(self) +│ │ ├── async fn execute: pub(self) +│ │ └── async fn schedule: pub(self) +│ ├── mod peer_discovery: pub +│ │ ├── enum FanoutError: pub +│ │ ├── struct Stage: pub +│ │ │ └── fn new: pub +│ │ └── struct Worker: pub +│ │ ├── async fn bootstrap: pub(self) +│ │ ├── async fn execute: pub(self) +│ │ └── async fn schedule: pub(self) +│ └── async fn run: pub +├── mod queue: pub(crate) +│ ├── struct Config: pub +│ ├── mod chaining: pub +│ │ ├── type LockEvent: pub(self) +│ │ └── struct TxChaining: pub +│ │ ├── fn is_chained_queue: pub +│ │ ├── async fn is_valid_token: pub +│ │ ├── async fn lock: pub +│ │ ├── fn new: pub +│ │ └── async fn unlock: pub +│ └── mod priority: pub +│ └── struct Priority: pub +│ ├── fn new: pub +│ ├── async fn next: pub +│ └── fn quota: pub(self) +├── mod server: pub(crate) +│ ├── struct Config: pub +│ ├── async fn run: pub +│ └── mod submit: pub(self) +│ └── struct SubmitServiceImpl: pub +│ └── fn new: pub +├── mod signing: pub(crate) +│ ├── struct Config: pub +│ ├── struct Secret: pub(self) +│ ├── trait SigningAdapter: pub +│ ├── enum SigningError: pub +│ ├── mod hashicorp: pub +│ │ └── struct HashicorpVaultClient: pub +│ │ ├── async fn get_mnemonic: pub(self) +│ │ ├── fn new: pub +│ │ ├── async fn sign: pub(self) +│ │ └── async fn verify: pub(self) +│ └── mod key: pub +│ ├── mod derive: pub +│ │ ├── fn from_bip39_mnenomic: pub(self) +│ │ ├── fn generate_account_key: pub +│ │ ├── fn generate_address: pub +│ │ ├── fn generate_address_with_delegation: pub +│ │ ├── fn generate_delegation_keypair: pub +│ │ ├── fn generate_payment_keypair: pub +│ │ ├── fn get_ed25519_keypair: pub +│ │ └── fn to_ed25519_keypair: pub +│ └── mod sign: pub +│ ├── fn sign_transaction: pub +│ └── fn to_built_transaction: pub +├── mod storage: pub(crate) +│ ├── struct Config: pub +│ ├── struct Cursor: pub +│ │ ├── fn from_row: pub(self) +│ │ └── fn new: pub +│ ├── struct Transaction: pub +│ │ ├── fn from_row: pub(self) +│ │ └── fn new: pub +│ ├── struct TransactionState: pub +│ │ └── fn from_row: pub(self) +│ ├── enum TransactionStatus: pub +│ └── mod sqlite: pub +│ ├── struct SqliteCursor: pub +│ │ ├── async fn current: pub +│ │ ├── fn new: pub +│ │ └── async fn set: pub +│ ├── struct SqliteStorage: pub +│ │ ├── async fn migrate: pub +│ │ └── async fn new: pub +│ └── struct SqliteTransaction: pub +│ ├── async fn create: pub +│ ├── async fn find: pub +│ ├── async fn find_to_rollback: pub +│ ├── async fn latest: pub +│ ├── fn new: pub +│ ├── async fn next: pub +│ ├── async fn state: pub +│ ├── async fn update: pub +│ └── async fn update_batch: pub +└── mod validation: pub(crate) + ├── async fn evaluate_tx: pub + └── async fn validate_tx: pub diff --git a/.cairn/context-map.md b/.cairn/context-map.md new file mode 100644 index 0000000..eb51191 --- /dev/null +++ b/.cairn/context-map.md @@ -0,0 +1,31 @@ +# Context map + + + +```mermaid +flowchart TD + pipeline -->|customer-supplier| storage + pipeline -->|customer-supplier| queue + pipeline -->|customer-supplier| signing + pipeline -->|customer-supplier| network + pipeline -->|conformist| ledger + queue -->|customer-supplier| storage + storage:::openhost + classDef openhost stroke-dasharray: 5 5 +``` + +| Consumer | Relationship | Supplier | Declared in | +|---|---|---|---| +| `queue` | `customer-supplier` | `storage` | `src/queue/MODULE.md` | +| `pipeline` | `customer-supplier` | `storage` | `src/pipeline/MODULE.md` | +| `pipeline` | `customer-supplier` | `queue` | `src/pipeline/MODULE.md` | +| `pipeline` | `customer-supplier` | `signing` | `src/pipeline/MODULE.md` | +| `pipeline` | `customer-supplier` | `network` | `src/pipeline/MODULE.md` | +| `pipeline` | `conformist` | `ledger` (u5c vocabulary) | `src/pipeline/MODULE.md` | +| `storage` | `open-host` (supplier-side declaration) | — | `src/storage/MODULE.md` | + +Modules without manifests yet (`ledger`, `network`, `signing`, `server`, `validation`) appear only as referenced suppliers; their consumer-side declarations are pending their own charter. + +## Expectations + +No manifest declares `Expectations` yet, so there are no consumer → provider links and the reverse index by provider is empty. Collaborator obligations are recorded only as gaps in each manifest's `Boundary allocation`. diff --git a/.cairn/crate-graph.svg b/.cairn/crate-graph.svg new file mode 100644 index 0000000..e2fdb62 --- /dev/null +++ b/.cairn/crate-graph.svg @@ -0,0 +1,18 @@ + + + + + + + + + +0 + +boros + + + diff --git a/.cairn/decisions/0001-return-gasket-to-published-release.md b/.cairn/decisions/0001-return-gasket-to-published-release.md new file mode 100644 index 0000000..9873093 --- /dev/null +++ b/.cairn/decisions/0001-return-gasket-to-published-release.md @@ -0,0 +1,15 @@ +# 0001 — Return gasket to a published release + +**Status:** accepted (human decision, 2026-08-23) + +## Context + +`Cargo.toml` pins `gasket` to a personal git fork (`construkts/gasket-rs`), a supply-chain and bus-factor liability surfaced by the census (`.cairn/shape.md`). + +## Decision + +The fork is a stopgap. Boros returns to a published upstream/crates.io `gasket` release as soon as one carries what the fork provides. + +## Consequences + +Until executed, the fork pin is visible debt: `external-policy = "review-new"` plus a future `cargo deny` sources check (SPEC §7.1) will keep flagging it. Executing this decision closes that finding. diff --git a/.cairn/decisions/0002-pin-toolchain-and-dependencies.md b/.cairn/decisions/0002-pin-toolchain-and-dependencies.md new file mode 100644 index 0000000..68c4bd3 --- /dev/null +++ b/.cairn/decisions/0002-pin-toolchain-and-dependencies.md @@ -0,0 +1,15 @@ +# 0002 — Pin toolchain and dependencies + +**Status:** proposed (finding from the first cairn survey, 2026-08-23; awaiting execution sign-off) + +## Context + +A clean checkout of `main` does not compile: `pallas = "1.0.0-alpha.2"` is a floating pre-release requirement that now resolves to `pallas 1.1.1`, whose u5c/pparams API differs from what the code was written against — and `Cargo.lock` is gitignored (the `.gitignore` template's library advice, applied to a binary), so nothing pins the resolution. The first gate run only succeeded after a local lockfile downgrade (`cargo update -p pallas --precise 1.0.0-alpha.2`). `rust-version` (MSRV) is also unpinned. + +## Decision (proposed) + +Commit `Cargo.lock`; pin `pallas = "=1.0.0-alpha.2"` exactly until upgraded deliberately; declare `rust-version` in `Cargo.toml`. + +## Consequences + +Builds become reproducible from a clean checkout; dependency upgrades become explicit diffs routed through the gate instead of silent re-resolution during dormancy. diff --git a/.cairn/decisions/0003-real-relay-source-for-peer-discovery.md b/.cairn/decisions/0003-real-relay-source-for-peer-discovery.md new file mode 100644 index 0000000..ddeeabd --- /dev/null +++ b/.cairn/decisions/0003-real-relay-source-for-peer-discovery.md @@ -0,0 +1,15 @@ +# 0003 — Peer discovery requires a real relay source + +**Status:** accepted (human decision, 2026-08-23) + +## Context + +Production wiring injects `MockRelayDataAdapter` into the peer-discovery stage (`src/pipeline/mod.rs:33`); the on-chain half of peer discovery is fake, leaving only config-listed peers and peer-sharing. + +## Decision + +This is a known gap, not intended design: a real on-chain relay data source must replace the mock before Boros is production-ready. + +## Consequences + +The `peer-discovery` flow doc notes the gap; the `ledger/relay` adapter trait is the implementation seam. Closing this decision removes the mock from `pipeline::run()`. diff --git a/.cairn/decisions/0004-fate-of-the-validated-status.md b/.cairn/decisions/0004-fate-of-the-validated-status.md new file mode 100644 index 0000000..a3647ed --- /dev/null +++ b/.cairn/decisions/0004-fate-of-the-validated-status.md @@ -0,0 +1,15 @@ +# 0004 — Fate of the `Validated` transaction status + +**Status:** open (deliberately undecided, 2026-08-23) + +## Context + +`TransactionStatus::Validated` is defined in `src/storage/mod.rs` and exercised only by a storage test; no production path ever sets it. The live lifecycle is `Pending → InFlight → {Confirmed, Failed}` (plus retry and rollback re-entries). A dead state in the lifecycle enum invites drift. + +## Decision + +None yet. Options: retire the variant, or wire it as a real stage (decoupling validation from broadcast in the ingest flow). + +## Consequences + +Until decided, INV-WS-001 documents the lifecycle without `Validated`, and this ADR is the pointer explaining the extra variant. diff --git a/.cairn/flows/confirm.md b/.cairn/flows/confirm.md new file mode 100644 index 0000000..e13e16a --- /dev/null +++ b/.cairn/flows/confirm.md @@ -0,0 +1,38 @@ ++++ +schema = "cairn/v0" +flow = "confirm" +short = "CONFIRM" +participants = ["pipeline", "ledger/u5c", "storage"] ++++ + +# Flow — confirm/rollback (monitor) + + + +The `monitor` gasket stage follows the chain through the u5c `ChainSyncStream` and drives the tail of the tx lifecycle: `InFlight → Confirmed` on inclusion, `InFlight → Pending` on retry timeout, and un-confirmation on rollback. The chain cursor is persisted after each event for crash recovery. + +**⚑ Flow-contract nominee (SPEC §12):** this flow's defining invariants are temporal and multi-module (crash recovery, rollback correctness, retry/confirm races). It is the survey's nomination for the first Quint model, if/when one is written. + +```mermaid +sequenceDiagram + participant L as ledger::u5c
ChainSyncStream + participant M as pipeline::monitor
Stage/Worker + participant DB as storage::sqlite
SqliteTransaction + participant CU as storage::sqlite
SqliteCursor + + L-->>M: Event::RollForward(Point, Vec) + M->>DB: find(InFlight) → Vec + M->>DB: update_batch(included ⇒ Confirmed, slot = block slot) + M->>DB: update_batch(stale beyond retry_slot_diff ⇒ Pending, slot = None) + L-->>M: Event::Rollback(Point) + M->>DB: find_to_rollback(slot) + M->>DB: update_batch(confirmed after slot ⇒ InFlight, slot = None) + M->>CU: set(Cursor { slot, hash }) — after status updates +``` + +## Invariants + +- **INV-CONFIRM-001** `[unverified]` — A transaction is `Confirmed` only when observed in a rolled-forward block; its `slot` then records the confirmation slot. +- **INV-CONFIRM-002** `[unverified]` — An `InFlight` transaction not confirmed within `retry_slot_diff` slots returns to `Pending` (and will be re-broadcast by fanout), so no transaction is stranded `InFlight` forever while the chain advances. +- **INV-CONFIRM-003** `[unverified]` — On rollback to slot S, transactions confirmed after S revert to `InFlight` and are eligible for re-confirmation; no transaction remains `Confirmed` at a slot beyond the new tip. +- **INV-CONFIRM-004** `[unverified]` — The cursor is persisted only after the corresponding status updates, so a crash between the two replays the event (at-least-once) rather than skipping it. diff --git a/.cairn/flows/fanout.md b/.cairn/flows/fanout.md new file mode 100644 index 0000000..8157fec --- /dev/null +++ b/.cairn/flows/fanout.md @@ -0,0 +1,51 @@ ++++ +schema = "cairn/v0" +flow = "fanout" +short = "FANOUT" +participants = ["pipeline", "queue", "signing", "validation", "ledger/u5c", "network", "storage"] ++++ + +# Flow — fanout (queue → peers) + + + +The `ingest` gasket stage drains `Pending` transactions by queue priority, optionally signs them server-side, re-validates, and broadcasts them to all connected Ouroboros tx-submission peers. Broadcast marks the tx `InFlight` stamped with the tip slot. + +```mermaid +sequenceDiagram + participant P as pipeline::ingest
Stage/Worker + participant PR as queue::priority
Priority + participant SG as signing
dyn SigningAdapter (Vault) + participant V as validation + participant L as ledger::u5c
dyn U5cDataAdapter + participant DB as storage::sqlite
SqliteTransaction + participant N as network
PeerManager → Peer/Mempool + + loop schedule (cap = 50 − queued) + P->>PR: next(Pending, cap) + PR->>DB: weighted pick per queue config + PR-->>P: Vec + end + loop per Transaction + opt queue.server_signing + P->>SG: sign(tx.raw) + SG-->>P: signed CBOR (Vec) + end + P->>V: validate_tx + evaluate_tx (MultiEraTx) + V->>L: ledger state + alt validation fails + P->>DB: update(status = Failed) + else ok + P->>N: broadcast Message> (gasket channel) + N->>N: Peer mempools serve ouroboros tx-submission + P->>L: fetch_tip() + P->>DB: update(status = InFlight, slot = tip) + end + end +``` + +## Invariants + +- **INV-FANOUT-001** `[unverified]` — A transaction reaches `InFlight` only after a successful broadcast, and its `slot` is set to the tip at broadcast time (this is what `monitor`'s retry window keys off). +- **INV-FANOUT-002** `[unverified]` — A transaction on a `server_signing` queue is never broadcast unsigned; absence of a signing adapter causes retry, not passthrough. +- **INV-FANOUT-003** `[unverified]` — In-flight output never exceeds the channel cap (50); scheduling backs off rather than dropping. diff --git a/.cairn/flows/peer-discovery.md b/.cairn/flows/peer-discovery.md new file mode 100644 index 0000000..b2a2c33 --- /dev/null +++ b/.cairn/flows/peer-discovery.md @@ -0,0 +1,32 @@ ++++ +schema = "cairn/v0" +flow = "peer-discovery" +short = "DISCOVERY" +participants = ["pipeline", "ledger/relay", "network"] ++++ + +# Flow — peer discovery + + + +The `peer_discovery` gasket stage tops up the peer pool toward `desired_peer_count`, choosing 50/50 between on-chain relays (via `RelayDataAdapter` — **currently the mock implementation**, an acknowledged gap: see `src/pipeline/mod.rs:33` and [decision 0003](../decisions/0003-real-relay-source-for-peer-discovery.md)) and peers learned from existing peers' peer-sharing. + +```mermaid +sequenceDiagram + participant R as ledger::relay
dyn RelayDataAdapter (MOCK) + participant D as pipeline::peer_discovery
Stage/Worker + participant PM as network::peer_manager
PeerManager + participant PE as network::peer
Peer (ouroboros) + + loop while connected < desired_peer_count + D->>R: get_relays() → Vec + D->>PM: pick_peer_rand(peers_per_request) → Option + Note over D: coin flip chooses relay vs peer-shared address + D->>PM: add_peer(addr) + PM->>PE: connect + start tx-submission protocol + end +``` + +## Invariants + +- **INV-DISCOVERY-001** `[unverified]` — The pool converges toward `desired_peer_count` and does not add peers beyond outstanding need (`peer_discovery_queue` bounds in-flight additions). diff --git a/.cairn/flows/submit.md b/.cairn/flows/submit.md new file mode 100644 index 0000000..7e795db --- /dev/null +++ b/.cairn/flows/submit.md @@ -0,0 +1,45 @@ ++++ +schema = "cairn/v0" +flow = "submit" +short = "SUBMIT" +participants = ["server", "validation", "ledger/u5c", "queue", "storage"] ++++ + +# Flow — submit (gRPC → queued) + + + +A client submits one or more raw transactions over gRPC. Each is decoded, optionally validated against ledger state, checked against queue lock tokens, and persisted as `Pending`. + +```mermaid +sequenceDiagram + participant C as Client (gRPC) + participant S as server::submit
SubmitServiceImpl + participant V as validation + participant L as ledger::u5c
dyn U5cDataAdapter + participant Q as queue::chaining
TxChaining + participant DB as storage::sqlite
SqliteTransaction + + C->>S: SubmitTxRequest { tx: [{raw, queue?, lock_token?}] } + S->>S: MultiEraTx::decode(raw) — err ⇒ failed_precondition (whole request) + alt queue is not server-signing + S->>V: validate_tx(&MultiEraTx) + V->>L: utxos / pparams + L-->>V: ledger state + S->>V: evaluate_tx(&MultiEraTx) + Note over S: validation/evaluation error ⇒ tx silently skipped (not in response) + end + opt chained queue + S->>Q: is_valid_token(queue, lock_token) + Q-->>S: false ⇒ permission_denied (whole request) + end + S->>DB: create(&[Transaction { status: Pending }]) + S->>Q: unlock(chained queues) + S-->>C: SubmitTxResponse { ref: [tx hashes] } +``` + +## Invariants + +- **INV-SUBMIT-001** `[unverified]` — A transaction accepted into storage always enters with status `Pending` and the queue name resolved (unknown queue falls back to the default queue). +- **INV-SUBMIT-002** `[unverified]` — A submission to a chained queue is persisted only if its lock token is valid; the queue is unlocked only after the batch is persisted. +- **INV-SUBMIT-003** `[unverified]` — Every hash in `SubmitTxResponse.ref` corresponds to a transaction that was persisted. *(Note the converse is currently false by design: txs failing validation are skipped silently and produce no error — an intentional-or-not behavior worth a human decision.)* diff --git a/.cairn/life-map.md b/.cairn/life-map.md new file mode 100644 index 0000000..bd06db3 --- /dev/null +++ b/.cairn/life-map.md @@ -0,0 +1,37 @@ +# Life map + + + +> **Deviation note:** the skill's 12-month churn window is empty — the repo's last commit is 2025-05-27 (~15 months before this survey). 42 of 44 lifetime commits landed Jan–May 2025. The partition below is derived from **lifetime** churn × complexity instead, and the whole repo carries a `dormant` flag. Whether dormancy demotes everything to fossil is a human judgment recorded after this survey. + +## Generated: churn × complexity (lifetime) + +Top files by churn (total lines added+deleted across history; renames noted): + +| File | Churn | Complexity (scc) | Note | +|---|---:|---:|---| +| `pipeline/ingest.rs` | 1,086 | 11 | churn leader | +| `storage/sqlite.rs` | 914 | 14 | | +| `network/peer.rs` | ~700 | **23** | complexity leader; churned as `pipeline/fanout/tx_submit_peer.rs` pre-rename | +| `ledger/u5c/mod.rs` | 580 | 12 | | +| `queue/chaining.rs` | 425 | 8 | | +| `network/mempool.rs` | ~314 | — | churned as `pipeline/fanout/mempool.rs` | +| `server/submit.rs` | 225 | 11 | | + +Commits per top module (lifetime): pipeline 25, server 12, storage 11, ledger 6, network 6, queue 3, signing 3, validation 2. + +## Curated: partition + +| Tier | Modules | Basis | +|---|---|---| +| **Active core** | `pipeline`, `storage`, `network`, `queue` | high churn × complexity intersection; `network` inherits the churn of its pre-rename `fanout` history | +| **Stable periphery** | `ledger`, `server` | moderate churn, adapter/API shells | +| **Thin periphery** | `signing`, `validation`, `main` | low churn, small, single-purpose | +| **Fossil** | *(none within repo)* | repo is young; but **entire repo dormant since 2025-05** | + +## Chartering nomination + +Per the adoption path (charter the 2–3 active-core modules first): **`pipeline`**, **`storage`**, **`queue`** — with `network` as the fourth if appetite allows (highest complexity, weakest tests: 1 test function, and it's in the mock). + +Change-coupling analysis (code-maat): **pending — tool not installed**; noted per skill guardrail. +Age strata (git-of-theseus): **pending — tool not installed**. diff --git a/.cairn/shape.md b/.cairn/shape.md new file mode 100644 index 0000000..bf63b13 --- /dev/null +++ b/.cairn/shape.md @@ -0,0 +1,44 @@ +# Shape (census) + + + +## Generated numbers + +### Size per module (tokei, lines of code) + +| Module | Code lines | Files | +|---|---:|---:| +| `network` | 1,112 | 4 | +| `pipeline` | 907 | 5 | +| `storage` | 846 | 3 | +| `queue` | 642 | 4 | +| `ledger` | 501 | 5 | +| `server` | 305 | 3 | +| `signing` | 242 | 3 | +| `validation` | 96 | 1 | +| `main.rs` | 100 | 1 | +| **Total (src/)** | **3,972** | **27** | + +Generated `spec` crate (protobuf bindings, excluded from analysis): 1,950 code lines / 8 files. + +### Ratios + +- Comment lines in `src/`: 44 (~1.1% of code) — effectively uncommented. +- Test functions: 33, concentrated in 6 files: + `storage/sqlite.rs` (15), `queue/chaining.rs` (8), `pipeline/ingest.rs` (5), `queue/priority.rs` (3), `ledger/relay/mod.rs` (1), `network/mock_ouroboros_tx_submit_server.rs` (1). + No integration-test target; `test/` holds fixtures only (CBOR tx files, block data). +- `unsafe`: 1 expression in first-party code (`signing/key/derive.rs:85`, `SecretKeyExtended::from_bytes_unchecked`). + Transitive unsafe (cargo-geiger): **pending — not yet run** (full-graph compile deferred). + +### Dependency weight + +- 461 packages in the resolved graph for ~4k first-party lines (~8.7 lines per dependency package). +- Notable heavy subtrees: `tonic`/`prost` (gRPC), `sqlx` (SQLite), `pallas` 1.0.0-alpha.2 (Cardano primitives, `phase2` feature), `vaultrs` (HashiCorp Vault), `gasket` (pinned to a git fork: `construkts/gasket-rs`). + +## Interpretation (curated) + +Boros is a small, young service (v0.1.0, bin-only crate) whose weight is in its dependencies, not its own code. Three facts stand out: + +1. **Test coverage is bimodal.** `storage` and `queue` are meaningfully tested; `network`, `server`, `signing`, `validation` are essentially untested. The untested set includes the one `unsafe` block (key derivation) and all I/O boundaries. +2. **The comment ratio (~1%) means the code carries no embedded rationale.** Manifests and flow docs are the only place intent can live — chartering matters more than usual here. +3. **`gasket` is pinned to a personal git fork**, a supply-chain and bus-factor liability — decision recorded in [decisions/0001](decisions/0001-return-gasket-to-published-release.md). diff --git a/.cairn/topology.md b/.cairn/topology.md new file mode 100644 index 0000000..7324516 --- /dev/null +++ b/.cairn/topology.md @@ -0,0 +1,50 @@ +# Topology + + + +> **Deviation note:** SPEC §2 expects crate-level topology. Boros is a single bin crate + one generated `spec` crate, so the crate graph (`crate-graph.svg`) is trivial. This file carries the *module-level* topology, which is where the structure actually lives. + +## Generated: module dependency edges + +Top-level module edges (submodules collapsed into parents; derived from `cargo modules dependencies`): + +```text +main → ledger, network, pipeline, queue, server, storage +pipeline → main, ledger, network, queue, signing, storage, validation +server → main, ledger, queue, storage, validation +queue → storage +validation → ledger +ledger, network, signing, storage → (no internal deps) +``` + +## Generated: instability table (module level) + +Ca = afferent (modules depending on it), Ce = efferent (internal modules it depends on), I = Ce/(Ca+Ce). + +| Module | Ca | Ce | I | Reading | +|---|---:|---:|---:|---| +| `ledger` | 4 | 0 | 0.00 | maximally stable | +| `storage` | 4 | 0 | 0.00 | maximally stable | +| `network` | 2 | 0 | 0.00 | stable | +| `signing` | 1 | 0 | 0.00 | stable | +| `queue` | 3 | 1 | 0.25 | stable-ish | +| `validation` | 2 | 1 | 0.33 | stable-ish | +| `main` (root) | 2 | 6 | 0.75 | orchestrator — **but Ca should be 0** | +| `server` | 1 | 5 | 0.83 | orchestrator | +| `pipeline` | 1 | 7 | 0.88 | orchestrator | + +## Generated: cycle detection + +One cycle at module level, through the crate root: + +- `main → pipeline → main` (and `main → server → main`): `Config` is defined in `main.rs` (`src/main.rs:72`) and imported by `pipeline::ingest` (`src/pipeline/ingest.rs:12`). The composition root is also a shared-type supplier. + +No cycles among the eight domain modules per `cargo-modules` — the domain graph is a clean DAG. **Tooling caveat:** `cargo-modules` misses const-only uses; source text shows `storage/mod.rs:6` importing `queue::DEFAULT_QUEUE`, making `storage ⇄ queue` a weak two-way coupling (queue → storage is the load-bearing direction; storage → queue is one default-value const). + +## Curated: layer labels (confirmed by human, 2026-08-23) + +| Layer | Modules | Intent | +|---|---|---| +| **Orchestration** | `main`, `pipeline`, `server` | composition root; gasket stage graph; gRPC API | +| **Domain** | `queue`, `validation` | tx queueing policy (priority, chaining); tx validation | +| **Foundation / adapters** | `ledger`, `network`, `storage`, `signing` | chain access (u5c/relay), Ouroboros peers, SQLite, key/Vault signing | diff --git a/WORKSPACE.md b/WORKSPACE.md new file mode 100644 index 0000000..f413700 --- /dev/null +++ b/WORKSPACE.md @@ -0,0 +1,119 @@ ++++ +schema = "cairn/v0" +id = "boros" +short = "WS" +purpose-status = "inferred" + +[workspace] +verify-all = "just gate" + +[process] +mode = "report-only" +auto-accept = { max-radius = 2, hotspot-overlap = false, api-delta = false } +sample-rate = 0.15 +mandatory = ["hotspot-overlap", "unsafe-touched", "tx-status-semantics-touched"] ++++ + +# Boros — workspace manifest + +## Purpose + +### System contribution + +Boros is a Cardano transaction submission service ("tx omnivore"): accept raw +transactions over gRPC, queue them with priority and chaining semantics, fan +them out to Ouroboros tx-submission peers, and settle their status against the +chain (confirm, retry, rollback). See `.cairn/README.md`. + +- OUT-WS-001: A submitter that hands Boros a fully-formed transaction has it delivered to the Cardano network without running its own node connections. +- OUT-WS-002: Submitters control the relative order of their submissions: weighted queues share dispatch capacity, and chained queues serialize dependent transactions. +- OUT-WS-003: An accepted transaction is pursued until it settles on-chain: rebroadcast while not included, confirmed when included, reopened when a rollback removes it. + +Whether submitters must be able to observe settlement is unknown: the UtxoRPC +`WaitForTx` and `WatchMempool` methods are unimplemented, and no status query +exists. + +### Beneficiaries + +| Beneficiary | Connection to outcomes | Provenance | +|---|---|---| +| Clients of the `boros.v1` submit service | Submit transactions (OUT-WS-001) and use queues and chain locks (OUT-WS-002) | Observed service surface; actual clients (dApp backends, wallets) are inferred | +| Clients of the UtxoRPC submit service | Submit transactions to the default queue (OUT-WS-001) | Observed service surface | +| Operators | Configure queues, peers, the UtxoRPC endpoint and Vault signing; run the service | Observed config schema; operational needs are unknown | +| Cardano network peers | Receive transactions over tx-submission | Observed protocol use | + +### Responsibilities + +| ID | Outcome | Responsibility | Realization | Coverage | +|---|---|---|---|---| +| RESP-WS-001 | OUT-WS-001 | Accept, decode, optionally validate and durably queue submitted transactions | `server` (no manifest; flow claims INV-SUBMIT-001..003 unverified); storage: RESP-STORE-001 | partial | +| RESP-WS-002 | OUT-WS-002 | Order dispatch across weighted queues and serialize chained queues | queue: RESP-QUEUE-001, RESP-QUEUE-002, RESP-QUEUE-003; pipeline: RESP-PIPE-001 | partial | +| RESP-WS-003 | OUT-WS-001 | Sign for server-signing queues, re-validate and broadcast to peers | pipeline: RESP-PIPE-001, RESP-PIPE-003; `signing`, `validation`, `network` have no manifests | partial | +| RESP-WS-004 | OUT-WS-003 | Settle transactions against the chain and resume after restart | pipeline: RESP-PIPE-002; storage: RESP-STORE-003, RESP-STORE-004 | uncovered | +| RESP-WS-005 | OUT-WS-001, OUT-WS-003 | Own the transaction lifecycle | INV-WS-001; decision 0004 leaves `Validated` open | partial | + +### Owned information + +| Information | Authority and representation ownership | +|---|---| +| Transaction lifecycle (status meaning and legal transitions) | Authoritative here (INV-WS-001); `storage` owns the representation | +| Service configuration (`boros.toml`, `/etc/boros/config.toml`, `BOROS_*` environment) | Root `Config` in `main.rs`; each module owns its section's schema | + +### Boundary allocation + +| Responsibility | This module | Collaborator obligation and owner | Contract gap | +|---|---|---|---| +| RESP-WS-001, RESP-WS-003 | Validate against current ledger state | A UtxoRPC endpoint (external, operator-chosen) supplies protocol parameters, UTxOs, tip and chain events | No workspace adapter manifest for `ledger` | +| RESP-WS-003 | Sign for server-signing queues | Hashicorp Vault (external) holds keys | No adapter manifest for `signing` | +| RESP-WS-003 | Broadcast | Cardano peers (external) accept tx-submission; a relay source supplies candidates: `unallocated` (mocked, decision 0003) | No manifest for `network` | +| RESP-WS-001 | Accept fully-formed transactions | Submitters build, balance and (except on server-signing queues) sign them | Stated as a non-goal | + +### Operating envelope + +| Responsibility | Mode / condition | Scope | Scenarios / gaps | +|---|---|---|---| +| RESP-WS-001 | Submission over either gRPC service | in-scope | [submit flow](.cairn/flows/submit.md) | +| RESP-WS-002, RESP-WS-003 | Steady-state dispatch | in-scope | [fanout flow](.cairn/flows/fanout.md) | +| RESP-WS-004 | Chain advance, retry, rollback | in-scope | [confirm flow](.cairn/flows/confirm.md) | +| RESP-WS-004 | Restart or crash recovery | in-scope | Cursor resume is observed; INV-CONFIRM-004 is unverified | +| RESP-WS-003 | Peer discovery from real relays | unknown | Mocked (decision 0003) | +| RESP-WS-001, RESP-WS-004 | UtxoRPC endpoint, Vault or peers unavailable | unknown | No accepted degraded behavior | +| RESP-WS-002 | Several Boros instances sharing one database | unknown | Chain locks are per process | +| RESP-WS-005 | `Validated` status | unknown | Decision 0004 | +| RESP-WS-001 | Building or balancing transactions | excluded | [Non-goals](#non-goals); owned by submitters | + +## Composition + +| Child | Manifest | Allocation | Exposed surface | Integration | +|---|---|---|---|---| +| pipeline | [pipeline](src/pipeline/MODULE.md) | RESP-WS-002 → RESP-PIPE-001; RESP-WS-003 → RESP-PIPE-001, RESP-PIPE-003; RESP-WS-004 → RESP-PIPE-002 | `pipeline::run` (internal) | Runs beside `server` in `main`; reads and writes through `storage`, schedules through `queue` | +| storage | [storage](src/storage/MODULE.md) | RESP-WS-001 → RESP-STORE-001; RESP-WS-004 → RESP-STORE-003, RESP-STORE-004 | `storage::{Transaction, Cursor}`, `storage::sqlite::{SqliteStorage, SqliteTransaction, SqliteCursor}` (internal) | One SQLite pool shared by all writers; no cross-module transaction | +| queue | [queue](src/queue/MODULE.md) | RESP-WS-002 → RESP-QUEUE-001, RESP-QUEUE-002, RESP-QUEUE-003 | `queue::{Config, priority::Priority, chaining::TxChaining}` (internal) | Shared by `server` and `pipeline`; lock state lives in one process | + +`server`, `network`, `ledger`, `signing`, `validation` and `main` have no +manifests yet; their work stays under this manifest and appears above as gaps. +No joint transaction spans the children. + +## Invariants + +- **INV-WS-001** `[llm-judged]` — All `Transaction.status` mutations flow through the `storage` write API; no module manipulates persisted status by other means. The legal lifecycle is `Pending → InFlight → {Confirmed, Failed}`, plus `InFlight → Pending` (retry) and `Confirmed → InFlight` (rollback). *(`Validated` exists in the enum but no production path uses it — see `.cairn/decisions/0004`.)* + - Scope and observation: every write of a persisted transaction status, across the workspace. + +## Non-goals + +- Boros does not build or balance transactions; it accepts fully-formed CBOR. +- Boros is not a general mempool: it tracks only transactions it accepted. + +## Language + +*queue* (named submission lane with weight), *chained queue* (lane serializing +dependent txs via lock tokens), *fanout* (broadcast to peers), *InFlight* +(broadcast, awaiting on-chain observation), *cursor* (last processed chain +point), *tip* (latest known chain point), *u5c* (UtxoRPC chain access). + +## Notes + +Open workspace-level concerns live as ADRs, not constraints: gasket git-fork +pin (`.cairn/decisions/0001`, accepted), toolchain/dependency pinning — clean +checkouts of `main` currently do not compile (`.cairn/decisions/0002`, +proposed), dead `Validated` status (`.cairn/decisions/0004`, open). diff --git a/justfile b/justfile new file mode 100644 index 0000000..2b1921d --- /dev/null +++ b/justfile @@ -0,0 +1,27 @@ +set shell := ["bash", "-o", "pipefail", "-cu"] + +# Cairn verifier verbs (SPEC §7): exit 0 = pass, stdout = evidence. +# Every verb here is referenced by a manifest; renaming one without +# updating its references is a lint error (L5). + +# Run all bound constraints (workspace verify-all) +gate: check-deps-all test-storage test-queue test-ingest + @echo "gate: all bound constraints passed" + +# Check a module's internal deps against its MODULE.md allowlist +check-deps id: + python3 tools/cairn_check_deps.py {{id}} + +check-deps-all: + python3 tools/cairn_check_deps.py storage + python3 tools/cairn_check_deps.py queue + python3 tools/cairn_check_deps.py pipeline + +test-storage: + cargo test storage:: -- --nocapture 2>&1 | tail -20 + +test-queue: + cargo test queue:: -- --nocapture 2>&1 | tail -20 + +test-ingest: + cargo test ingest_tests:: -- --nocapture 2>&1 | tail -20 diff --git a/src/pipeline/MODULE.md b/src/pipeline/MODULE.md new file mode 100644 index 0000000..252da74 --- /dev/null +++ b/src/pipeline/MODULE.md @@ -0,0 +1,115 @@ ++++ +schema = "cairn/v0" +id = "pipeline" +short = "PIPE" +role = "core-domain" +purpose-status = "inferred" +parnas-secret = "the runtime topology — which gasket stages exist, how they are scheduled, and how backpressure (channel cap) is applied" + +[deps] +internal-allowed = ["ledger", "network", "queue", "signing", "storage", "validation"] +external-policy = "review-new" +verify = "just check-deps pipeline" ++++ + +# pipeline — module manifest + +## Purpose + +### System contribution + +The orchestrator: three gasket stages wired in `run()` — `ingest` (drain +Pending by priority → sign → validate → broadcast → InFlight), `monitor` +(chainsync → Confirmed / retry / rollback + cursor), `peer_discovery` (peer +pool top-up). Contributes to the workspace outcomes +[OUT-WS-001 and OUT-WS-003](../../WORKSPACE.md#system-contribution): queued +transactions reach the network and are pursued until settled. Temporal +behavior across these stages is specified by the flow docs `fanout`, +`confirm` and `peer-discovery` (see `.cairn/flows/`). + +### Beneficiaries + +| Beneficiary | Connection to outcomes | Provenance | +|---|---|---| +| `main` | Starts the pipeline beside the gRPC server | Observed caller of `run` | +| Submitters | Their queued transactions are broadcast and settled (OUT-WS-001, OUT-WS-003) | Inferred indirect beneficiaries | +| Cardano peers | Receive transactions over Ouroboros tx-submission | Observed through `network`; peer-side needs are unknown | +| Operators | Configure peers, retry window and signing | Observed config; operational needs (metrics, health) are unknown | + +### Responsibilities + +| ID | Outcome | Responsibility | Realization | Coverage | +|---|---|---|---|---| +| RESP-PIPE-001 | OUT-WS-001 | Drain pending transactions, sign those on server-signing queues, re-validate, broadcast, and mark each `InFlight` or `Failed` | INV-PIPE-001, INV-PIPE-002; flow claims INV-FANOUT-001..003 are unverified | partial | +| RESP-PIPE-002 | OUT-WS-003 | Follow the chain and settle in-flight transactions: confirm, retry, revert on rollback, persist the cursor | No module guarantee; flow claims INV-CONFIRM-001..004 are unverified | uncovered | +| RESP-PIPE-003 | OUT-WS-001 | Keep the tx-submission peer pool topped up | Flow claim INV-DISCOVERY-001 is unverified; the relay source is mocked (decision 0003) | uncovered | +| RESP-PIPE-004 | OUT-WS-001, OUT-WS-003 | Own the runtime topology: stage wiring, scheduling, failure policy and backpressure | INV-PIPE-002, INV-PIPE-003; INV-FANOUT-003 is unverified | partial | + +### Owned information + +| Information | Authority and representation ownership | +|---|---| +| Stage topology and the broadcast channel cap (50) | Authoritative, in code | +| Monitor retry window (`retry_slot_diff`) | Owns the config schema; operators supply the value | +| Transaction status and cursor values | Decides the transitions it writes; `storage` owns their representation and the workspace owns the lifecycle (INV-WS-001) | + +### Boundary allocation + +| Responsibility | This module | Collaborator obligation and owner | Contract gap | +|---|---|---|---| +| RESP-PIPE-001 | Choose, sign, validate and dispatch | `queue` picks the batch; `signing` signs (Hashicorp Vault, external); `validation` judges against ledger state from `ledger` (UtxoRPC endpoint, external) | `signing`, `validation`, `ledger` have no manifests; no expectations declared | +| RESP-PIPE-001 | Put the transaction on the broadcast channel | `network` delivers it to connected peers | Delivery is not acknowledged back; `InFlight` means sent to the channel | +| RESP-PIPE-002 | Map chain events to status updates and the cursor | `ledger::u5c` supplies roll-forward and rollback events in order from the cursor; `storage` persists | Ordering and gap-freedom of the event stream are not declared | +| RESP-PIPE-003 | Schedule pool top-up | `network::PeerManager` connects; a relay source supplies candidates: `unallocated` (mock in use) | Decision 0003 | + +### Operating envelope + +| Responsibility | Mode / condition | Scope | Scenarios / gaps | +|---|---|---|---| +| RESP-PIPE-001 | Normal drain and broadcast | in-scope | [fanout flow](../../.cairn/flows/fanout.md) | +| RESP-PIPE-001 | Transaction fails validation or evaluation | in-scope | Marked `Failed` (INV-PIPE-002) | +| RESP-PIPE-001 | Transaction that does not decode, or whose queue is missing from config | unknown | The stage errors or retries the whole batch; whether that wedges the stage is unassessed | +| RESP-PIPE-001 | Server-signing queue with no signer configured | in-scope | Retry, never unsigned broadcast (INV-FANOUT-002, unverified) | +| RESP-PIPE-002 | Roll-forward, retry timeout, rollback | in-scope | [confirm flow](../../.cairn/flows/confirm.md); flow-contract nominee | +| RESP-PIPE-002 | Crash between status updates and cursor write | in-scope | INV-CONFIRM-004, unverified | +| RESP-PIPE-003 | Peer pool below target | in-scope | [peer-discovery flow](../../.cairn/flows/peer-discovery.md); relay side mocked | +| RESP-PIPE-004 | UtxoRPC endpoint or all peers unavailable | unknown | Gasket retry policy is the default; no accepted behavior | + +## Guarantees + +- **INV-PIPE-001** `[bound → just test-ingest]` — Phase-1 validation and phase-2 evaluation accept known-valid Conway transactions and reject invalid ones (fixture-backed, `ingest_tests`). + - Surface: `validation::{validate_tx, evaluate_tx}` as called by the `ingest` stage. + - When: Ledger state comes from the test UtxoRPC mock; fixtures are Conway transactions. + - Then: The valid fixture passes both; the unwitnessed fixture fails validation; the bad-script fixture fails evaluation. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. The functions live in `validation`, which has no manifest. +- **INV-PIPE-002** `[llm-judged]` — Stage failures are contained by gasket retry policy; a failing transaction is marked `Failed` rather than wedging the stage (refines INV-WS-001). + - Surface: the `ingest` stage. + - When: A transaction fails validation or evaluation. + - Then: It is written `Failed` and the stage continues with the next transaction. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. + +## Invariants + +- **INV-PIPE-003** `[unverified]` — Temporal invariants of this module's stages are owned by the flow docs (INV-FANOUT-*, INV-CONFIRM-*, INV-DISCOVERY-*); this entry records that they are not yet verified anywhere. + - Scope and observation: every execution of the three stages. + +## Relationships + +- `customer-supplier` with **storage**, **queue**, **signing** (pipeline is the customer). +- `conformist` toward **ledger** (u5c): pipeline consumes the UtxoRPC event/point vocabulary as-is, no translation layer. +- `customer-supplier` with **network** (pipeline feeds the broadcast channel network peers consume). + +## Non-goals + +- No policy decisions: what to pick (queue), what is valid (validation), how to persist (storage) are supplied, not owned. + +## Notes + +`run()` currently wires `MockRelayDataAdapter` into peer discovery +(`src/pipeline/mod.rs:33`) — an acknowledged gap, see `.cairn/decisions/0003`. +The `main ⇄ pipeline` cycle through the root `Config` type is documented in +`.cairn/topology.md`; moving shared config out of `main.rs` is an open idea, +not a decision. + +INV-PIPE-003 is a pointer rather than an enduring constraint; it is kept +because IDs are immutable, and retiring it is an owner's call. diff --git a/src/queue/MODULE.md b/src/queue/MODULE.md new file mode 100644 index 0000000..f91269c --- /dev/null +++ b/src/queue/MODULE.md @@ -0,0 +1,111 @@ ++++ +schema = "cairn/v0" +id = "queue" +short = "QUEUE" +role = "core-domain" +purpose-status = "inferred" +parnas-secret = "the scheduling policy — how weighted priority picks the next transactions, and how chained queues serialize dependent submissions via lock tokens" + +[deps] +internal-allowed = ["storage"] +external-policy = "review-new" +verify = "just check-deps queue" ++++ + +# queue — module manifest + +## Purpose + +### System contribution + +The scheduling heart of Boros. Contributes to the workspace outcome +[OUT-WS-002](../../WORKSPACE.md#system-contribution): submitters control the +relative order of their submissions. `priority` allocates each dispatch batch +across named queues by weight; `chaining` gives dependent-transaction queues +exclusive-lock semantics (token-gated submission, streamed lock state). + +### Beneficiaries + +| Beneficiary | Connection to outcomes | Provenance | +|---|---|---| +| `pipeline::ingest` | Asks `Priority::next` for each dispatch batch (OUT-WS-002) | Observed caller | +| `server` (boros submit service) | Locks, validates tokens on and unlocks chained queues (OUT-WS-002) | Observed caller of `TxChaining` | +| Submitters building transaction chains | Need to read the latest queued transaction and append the next one without a competing writer | Inferred from the `LockState` RPC and the token check; no client was inspected | +| Operators | Declare queues, weights, chaining and server signing in config | Observed config schema; operator needs beyond that are unknown | + +### Responsibilities + +| ID | Outcome | Responsibility | Realization | Coverage | +|---|---|---|---|---| +| RESP-QUEUE-001 | OUT-WS-002 | Allocate each dispatch batch across queues in proportion to their weights | INV-QUEUE-001, INV-QUEUE-002. INV-QUEUE-001's claim that the cap is fully distributed is contradicted by the code: rounding can over- or undershoot the cap, and leftover from the last queue visited is dropped. Fairness across successive batches is not declared. | partial | +| RESP-QUEUE-002 | OUT-WS-002 | Serialize submissions to chained queues through exclusive, expiring lock tokens | INV-QUEUE-003, INV-QUEUE-004. Lock state is held in memory only. | partial | +| RESP-QUEUE-003 | OUT-WS-002 | Define queue identity and the default queue | INV-QUEUE-005 | partial | + +### Owned information + +| Information | Authority and representation ownership | +|---|---| +| Queue configuration (`name`, `weight`, `chained`, `server_signing`) and the `default` queue | Owns the schema and identity rules; operators supply values; `main` inserts the default queue when absent | +| Chained-queue lock tokens | Authoritative, in memory, per process; lost on restart | +| Per-queue batch quotas | Derived per call from storage counts; not persisted | + +### Boundary allocation + +| Responsibility | This module | Collaborator obligation and owner | Contract gap | +|---|---|---|---| +| RESP-QUEUE-001 | Compute per-queue limits | `storage` returns counts and selections honoring them (its INV-STORE-002) | No expectation declared | +| RESP-QUEUE-001 | Hand back the batch | `pipeline::ingest` dispatches it; it retries a transaction whose queue is missing from config | Interaction with INV-QUEUE-002 is unassessed | +| RESP-QUEUE-002 | Issue, check and expire tokens | `server` checks the token before `create` and unlocks after it | `server` has no manifest; no expectation declared | +| RESP-QUEUE-002 | Stream the latest queued transaction with the token | `storage::latest` returns the most recent submission in the queue | It may return a `Failed` transaction (TODO in source) | + +### Operating envelope + +| Responsibility | Mode / condition | Scope | Scenarios / gaps | +|---|---|---|---| +| RESP-QUEUE-001 | Normal drain across configured queues | in-scope | [fanout flow](../../.cairn/flows/fanout.md) | +| RESP-QUEUE-001 | Queue removed from config while it holds transactions | in-scope | INV-QUEUE-002; downstream signing lookup unassessed | +| RESP-QUEUE-001 | Weights do not split the cap into whole shares | in-scope | Gap: shares are rounded independently, so the batch can exceed or fall short of the cap; no claim or test covers it | +| RESP-QUEUE-001 | A queue holds fewer transactions than its share | in-scope | Gap: unused capacity passes to queues in hash order, and the last queue's leftover is dropped, so the batch size depends on that order; no test covers it | +| RESP-QUEUE-001 | Starvation of low-weight queues under sustained load | unknown | No claim or scenario | +| RESP-QUEUE-002 | Lock, submit, unlock within the timeout | in-scope | [submit flow](../../.cairn/flows/submit.md) | +| RESP-QUEUE-002 | Lock holder exceeds the 30 s timeout | in-scope | Lock expires; a late submission is rejected (INV-QUEUE-004) | +| RESP-QUEUE-002 | Process restart while a lock is held | unknown | Tokens are lost; intended behavior undecided | +| RESP-QUEUE-002 | Several Boros instances serving one queue | unknown | Locks are per process | +| RESP-QUEUE-003 | Config change at runtime | unknown | Config is read once at startup | + +## Guarantees + +- **INV-QUEUE-001** `[bound → just test-queue]` — Batch quota is allocated proportionally to queue weights and fully distributed (remainder included) (`it_should_calculate_quota`). + - Surface: `Priority::next` (quota computation). + - When: At least one queue holds transactions in the requested status. + - Then: Each queue's share is its weight's fraction of the cap, rounded to the nearest whole transaction; `it_should_calculate_quota` checks one exact split (weights 1/2/2, cap 10 → 2/4/4). Capacity a queue cannot use is added to the share of the queue visited after it. + - Contradiction: the claim's "fully distributed (remainder included)" does not hold. Rounding each share can exceed or fall short of the cap (weights 1/1, cap 5 → 3 + 3 = 6; weights 1/1/1, cap 10 → 9). Queues are visited in hash order and the last one's leftover is dropped (cap 50, two weight-1 queues holding 100 and 1 → a batch of 26 or 50). The claim and tier stay as declared: demoting needs a human-approved change (§4), and fixing the code is outside this charter. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-QUEUE-002** `[bound → just test-queue]` — Transactions in queues removed from config are still drained, not stranded (`it_should_return_next_transactions_when_a_queue_is_removed_from_config`). + - Surface: `Priority::next`. + - When: Stored transactions belong to a queue absent from config. + - Then: That queue is scheduled with the default weight. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-QUEUE-003** `[bound → just test-queue]` — A chained queue admits one lock holder at a time; competing lockers wait or time out (`it_should_lock_queue`, `it_should_wait_timeout_to_lock_queue`, `it_should_lock_many_queue`). + - Surface: `TxChaining::lock`. + - When: The queue is configured `chained`. + - Then: A second locker receives its token only after the first unlocks or its lock times out. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-QUEUE-004** `[bound → just test-queue]` — Only the current lock token is valid for submission to a chained queue; unlock invalidates it (`it_should_return_token_*`, `it_should_unlock_queue`). + - Surface: `TxChaining::{is_valid_token, unlock}`. + - When: The queue is configured `chained`. + - Then: `is_valid_token` is true only for the token most recently issued and not yet released or expired. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. + +## Invariants + +- **INV-QUEUE-005** `[llm-judged]` — Queue identity is its name alone (`Config` hashes/compares by name), so config reload with changed weights re-targets the same queue rather than creating a phantom. + - Scope and observation: every set of queue `Config` values, at construction. + +## Relationships + +- `customer-supplier` with **storage** (queue is the customer: priority and chaining read/write through storage's store types). + +## Non-goals + +- No transport, no validation, no signing: queue decides *order and admission*, nothing else. diff --git a/src/storage/MODULE.md b/src/storage/MODULE.md new file mode 100644 index 0000000..097446e --- /dev/null +++ b/src/storage/MODULE.md @@ -0,0 +1,115 @@ ++++ +schema = "cairn/v0" +id = "storage" +short = "STORE" +role = "generic-subdomain" +purpose-status = "inferred" +parnas-secret = "how transaction/cursor state is persisted — SQLite, the schema, and all SQL live only here" + +[deps] +internal-allowed = ["queue#const-only"] +external-policy = "review-new" +verify = "just check-deps storage" ++++ + +# storage — module manifest + +## Purpose + +### System contribution + +Within Boros, keep the submission state durable so the service can keep +pursuing every accepted transaction and resume following the chain where it +left off. Contributes to the workspace outcomes +[OUT-WS-001, OUT-WS-002 and OUT-WS-003](../../WORKSPACE.md#system-contribution): +accepted transactions are recorded, queues can be drained in order, and +settlement state survives restarts. Storage supplies representation and +queries; it does not decide lifecycle transitions. + +### Beneficiaries + +| Beneficiary | Connection to outcomes | Provenance | +|---|---|---| +| `server` (boros and UtxoRPC submit services) | Records submitted batches for OUT-WS-001 | Observed caller of `create` | +| `queue` (`priority`, `chaining`) | Reads per-queue counts, ordered selections and the latest submission for OUT-WS-002 | Observed caller of `state`, `next`, `latest` | +| `pipeline` (`ingest`, `monitor`) | Writes status transitions and the cursor for OUT-WS-001 and OUT-WS-003 | Observed caller of `update`, `update_batch`, `find`, `find_to_rollback`, `SqliteCursor` | +| Operators | Need a database that migrates in place and survives restarts | Inferred; durability and retention needs were not stated anywhere | + +### Responsibilities + +| ID | Outcome | Responsibility | Realization | Coverage | +|---|---|---|---|---| +| RESP-STORE-001 | OUT-WS-001 | Record accepted transactions with their queue, status and declared dependencies | INV-STORE-001. Batch atomicity of `create` is observed (one database transaction) but not declared; duplicate submissions are unassessed. | partial | +| RESP-STORE-002 | OUT-WS-002, OUT-WS-003 | Answer lifecycle queries: per-queue counts and ordered selection, the in-flight set, rollback candidates, the latest submission per queue | INV-STORE-002, INV-STORE-003. `find`, `state` and `latest` have tests but no declared guarantee. | partial | +| RESP-STORE-003 | OUT-WS-003 | Persist status transitions written by callers | `update` and `update_batch` exist; no guarantee declared. Transition legality is the workspace's INV-WS-001, not checked here. | uncovered | +| RESP-STORE-004 | OUT-WS-003 | Persist the chain-follow cursor | INV-STORE-004. Atomicity with the status updates of the same chain event is unallocated. | partial | +| RESP-STORE-005 | OUT-WS-001, OUT-WS-003 | Own the schema, its migrations and the confinement of SQL | INV-STORE-005, INV-STORE-006 | partial | + +### Owned information + +| Information | Authority and representation ownership | +|---|---| +| Transaction records (id = transaction hash hex, raw CBOR, status, queue, slot, timestamps) | Owns the `tx` table representation; the meaning of `status` belongs to the workspace lifecycle (INV-WS-001) | +| Dependency edges | Owns `tx_dependence`; written by `create`, never read back (`Transaction.dependencies` loads as `None`) | +| Chain-follow cursor | Owns the single-row `cursor` table; `pipeline` decides when it advances | +| Schema | Authoritative: `src/storage/migrations/` | + +### Boundary allocation + +| Responsibility | This module | Collaborator obligation and owner | Contract gap | +|---|---|---|---| +| RESP-STORE-001 | Insert the batch and its dependency edges | `server` decodes, validates and assigns queues before calling `create` | `server` has no manifest; no expectation declared | +| RESP-STORE-001, RESP-STORE-002 | Record dependency edges | Dispatch in dependency order: `unallocated` (`next` does not consult `tx_dependence`, and no other module does) | Whether dependencies are meant to order dispatch is unknown | +| RESP-STORE-002 | Execute selections with caller-supplied limits | `queue` computes per-queue limits | No expectation declared | +| RESP-STORE-003, RESP-STORE-004 | Apply writes as given | `pipeline` chooses legal transitions and writes the cursor after its status updates | Flow claim INV-CONFIRM-004 is unverified | +| RESP-STORE-005 | Run migrations when asked | The binary entry point (`main`) migrates before serving | No manifest owns `main`; see INV-STORE-006 | + +### Operating envelope + +| Responsibility | Mode / condition | Scope | Scenarios / gaps | +|---|---|---|---| +| RESP-STORE-001 | Submission batch with valid dependencies | in-scope | [submit flow](../../.cairn/flows/submit.md) | +| RESP-STORE-001 | Resubmission of a transaction already stored | unknown | The primary key rejects the whole batch; intended behavior undecided | +| RESP-STORE-002 | Draining pending transactions under load | in-scope | [fanout flow](../../.cairn/flows/fanout.md) | +| RESP-STORE-002, RESP-STORE-003 | Concurrent writers (`server`, `ingest`, `monitor`) on one pool | unknown | No isolation guarantee declared | +| RESP-STORE-003, RESP-STORE-004 | Crash between status updates and cursor write | unknown | [confirm flow](../../.cairn/flows/confirm.md), INV-CONFIRM-004 unverified | +| RESP-STORE-005 | Migrating an existing database | in-scope | No scenario or test beyond the in-memory database | +| RESP-STORE-001, RESP-STORE-003 | Retention of settled transactions | unknown | No deletion path exists; growth is unbounded | + +## Guarantees + +- **INV-STORE-001** `[bound → just test-storage]` — Creating transactions with declared dependencies fails unless every dependency is already present (`it_should_fail_create_with_invalid_dependencies`). + - Surface: `SqliteTransaction::create`. + - When: The database is migrated; a transaction in the batch declares `dependencies`. + - Then: `create` returns `Err` when a declared dependency id is not stored. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-STORE-002** `[bound → just test-storage]` — `next(status, quotas)` honors per-queue quotas and status filtering; transactions from queues no longer in config still drain (`it_should_find_next*`). + - Surface: `SqliteTransaction::next`. + - When: The caller supplies a status and a per-queue limit map. + - Then: The result holds only transactions in that status, at most the limit per listed queue, oldest first within a queue. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-STORE-003** `[bound → just test-storage]` — `find_to_rollback(slot)` returns only transactions whose recorded slot is affected by a rollback to `slot` (`it_should_find_to_rollback*`). + - Surface: `SqliteTransaction::find_to_rollback`. + - When: The caller supplies the rollback slot. + - Then: The result holds exactly the `Confirmed` transactions whose slot is after it. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. +- **INV-STORE-004** `[bound → just test-storage]` — Cursor `set` upserts: a second write updates rather than duplicates (`it_should_set_when_it_updates`). + - Surface: `SqliteCursor::{set, current}`. + - When: The database is migrated. + - Then: After `set` returns `Ok`, `current` returns that cursor; there is never more than one. + - Migration: moved from Invariants under SPEC §3.4; ID and tier unchanged. + +## Invariants + +- **INV-STORE-005** `[llm-judged]` — No SQL and no `sqlx` usage outside this module; consumers speak `Transaction`/`Cursor`, never rows. + - Scope and observation: the workspace source, at every revision. +- **INV-STORE-006** `[unverified]` — Migrations run to completion before any query is served (currently upheld only by call order in `main.rs`; nothing enforces it). + - Scope and observation: process lifetime, from startup to the first query. + +## Relationships + +- *(supplier)* `open-host` — storage is the shared persistence supplier for `queue`, `pipeline`, and `server`; its store types are the published surface. + +## Non-goals + +- No domain policy: which tx goes next (queue), when to retry (pipeline) are not storage's decisions. diff --git a/tools/cairn_check_deps.py b/tools/cairn_check_deps.py new file mode 100644 index 0000000..1cbffda --- /dev/null +++ b/tools/cairn_check_deps.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Bound verifier for MODULE.md [deps].internal-allowed (cairn SPEC §3.1). + +Derives the module's actual internal dependencies from `cargo modules +dependencies` and asserts they are a subset of the manifest allowlist. + +Known blind spots: cargo-modules does not report const-only uses (see +.cairn/topology.md); such edges must be policed by review. The frontmatter +reader understands only a single string array assigned to `internal-allowed` +inside `[deps]` (no tomllib before Python 3.11); any other form reads as an +empty allowlist, which fails closed. +""" + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +TOP_MODULES = { + "ledger", "network", "pipeline", "queue", + "server", "signing", "storage", "validation", +} + + +def manifest_allowlist(module_id: str) -> list[str]: + manifest = ROOT / "src" / module_id / "MODULE.md" + if not manifest.exists(): + sys.exit(f"FAIL: no manifest at {manifest}") + text = manifest.read_text() + m = re.match(r"\+\+\+\n(.*?)\n\+\+\+", text, re.DOTALL) + if not m: + sys.exit(f"FAIL: no +++ frontmatter in {manifest}") + deps = re.search(r"^\[deps\]\n(.*?)(?=^\[|\Z)", m.group(1), re.DOTALL | re.MULTILINE) + allowed = deps and re.search(r"^internal-allowed\s*=\s*\[(.*?)\]", deps.group(1), re.DOTALL | re.MULTILINE) + if not allowed: + return [] + return [d.split("#")[0] for d in re.findall(r'"([^"]*)"', allowed.group(1))] + + +def actual_deps(module_id: str) -> set[str]: + out = subprocess.run( + ["cargo", "modules", "dependencies", "--bin", "boros", + "--no-externs", "--no-sysroot"], + cwd=ROOT, capture_output=True, text=True, check=True, + ).stdout + deps: set[str] = set() + for src, dst in re.findall(r'"(boros[^"]*)" -> "(boros[^"]*)".*"uses"', out): + src_mod = (src.split("::") + [None])[1] + dst_mod = (dst.split("::") + [None])[1] + if src_mod == module_id and dst_mod in TOP_MODULES and dst_mod != module_id: + deps.add(dst_mod) + return deps + + +def main() -> None: + module_id = sys.argv[1] + allowed = set(manifest_allowlist(module_id)) + actual = actual_deps(module_id) + illegal = actual - allowed + print(f"module: {module_id}") + print(f"allowed: {sorted(allowed) or '(none)'}") + print(f"actual: {sorted(actual) or '(none)'}") + if illegal: + sys.exit(f"FAIL: undeclared internal deps: {sorted(illegal)}") + print("PASS: actual deps ⊆ allowlist") + + +if __name__ == "__main__": + main()