diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1d13b4..b05b3b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,11 @@ on: pull_request: jobs: + # Named for the status check main's branch protection requires. The + # duplicate workflow deleted in c11b779 carried this job name; removing it + # took the required context with it and nothing could merge afterwards. test: + name: Foundry project runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ledger-tests.yml b/.github/workflows/ledger-tests.yml index a58dd14..c3140b6 100644 --- a/.github/workflows/ledger-tests.yml +++ b/.github/workflows/ledger-tests.yml @@ -17,7 +17,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Test receipt, balance, and supply verification + # Every Python suite lives in script/; test/ holds Solidity only. + - name: Python tests run: python3 -B -m unittest discover -s script -p 'test_*.py' -v - - name: Test verifier integration - run: python3 -B -m unittest discover -s test -p 'test_*.py' -v diff --git a/.github/workflows/ledger.yml b/.github/workflows/ledger.yml index 79b146a..f4a2539 100644 --- a/.github/workflows/ledger.yml +++ b/.github/workflows/ledger.yml @@ -18,6 +18,7 @@ on: jobs: verify: name: Verify on-chain accounting + timeout-minutes: 150 runs-on: ubuntu-latest permissions: contents: read @@ -31,15 +32,25 @@ jobs: - name: Validate ledger shape and published figures run: | python3 script/ledger_schema.py - python3 -B -m unittest discover -s script -p 'test_ledger_schema.py' + python3 -B -m unittest discover -s script -p 'test_*.py' + # Proving nothing is MISSING means reading all ~16.3M blocks since + # deployment: over an hour against free endpoints, and growing. That is + # not a per-pull-request cost, so a PR confirms every entry is real and + # the balances add up (about a second), and the full scan runs on main + # and on the weekly schedule. The fast mode prints what it did not check. - name: Verify ledger against Base mainnet env: - # Nonces and balances. mainnet.base.org serves these and refuses - # eth_getLogs, so the log reads go elsewhere. Both public, no key. + BASE_VERIFY_SCOPE: ${{ github.event_name == 'pull_request' && 'entries' || 'full' }} + # Snapshot headers and state use the primary RPC; token logs use + # the logs RPC. Both endpoints must report the configured chain. BASE_RPC_URL: https://mainnet.base.org BASE_LOGS_RPC_URL: https://base.gateway.tenderly.co # Shared runner egress gets throttled where a laptop does not. BASE_RPC_MIN_INTERVAL: "0.25" + # A complete pass is ~16,300 pages since deployment, so at the pace + # above it needs well over an hour and grows with the chain. Give it + # room rather than let every run defer without verifying anything. + BASE_HISTORY_TIMEOUT: "7200" run: | set +e python3 script/verify_ledger.py @@ -49,9 +60,9 @@ jobs: echo "Ledger verified against Base mainnet." exit 0 elif [ "$code" -eq 2 ]; then - # An indexer outage is not an accounting finding. Each run re-checks - # the full transfer history, so a deferred check loses nothing. - echo "::warning title=Ledger not verified::Transfer data source unreachable; deferring to the next run." + # Incomplete coverage is not an accounting finding. Every run + # starts again from deployment and reports its actual coverage. + echo "::warning title=Ledger not verified::History or snapshot could not be verified within source/resource limits; deferring." exit 0 else echo "::error title=Accounting violation::A project-wallet movement is unaccounted for in rewards/ledger.json." diff --git a/Makefile b/Makefile index b1f0bc6..250cf4a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ fmt: test: forge test -vvv python3 -B -m unittest discover -s script -p 'test_*.py' - python3 -B -m unittest discover -s test -p 'test_*.py' verify: python3 -B script/ledger_schema.py diff --git a/README.md b/README.md index 9cb6a59..dec679b 100644 --- a/README.md +++ b/README.md @@ -55,3 +55,36 @@ Deployment artifacts: - Base Mainnet (8453): `0x914F6b6a0DaD39A3CB932dcf4D1af885C7c08EeB` (GA) - Verified: https://basescan.org/address/0x914f6b6a0dad39a3cb932dcf4d1af885c7c08eeb#code - Sepolia (11155111): `0x8c2B67EA395824Cc069F5e13b1d80E6016751a4A` (testnet) + +## Ledger verification coverage + +`python3 script/verify_ledger.py` scans this token's `Transfer` logs from its +configured deployment block through a fixed head. Both configured RPC endpoints +must report the ledger's chain ID and agree on that head. Balances and total +supply are read using its canonical block hash. This covers delegated +`transferFrom` movements even if balances later return to their original values +and allowances are revoked. Current allowances are not evidence of missing +historical transfers. + +The verifier trusts complete RPC log responses; this is an accounting check, +not an independent cryptographic receipt proof. It keeps the existing scope: +mint events and transfers out of project wallets must match the ledger. + +| Environment setting | Default | Purpose | +| --- | --- | --- | +| `BASE_HISTORY_BLOCK_SPAN` | `10000` | Maximum blocks per log request | +| `BASE_HISTORY_MAX_REQUESTS` | `2048` | Shared request budget, including wire retries and state reads | +| `BASE_HISTORY_TIMEOUT` | `300` | Deadline in seconds, checked around RPC operations and retry delays | +| `BASE_HISTORY_MAX_RESPONSE_BYTES` | `8388608` | Maximum response body per request | +| `BASE_HISTORY_MAX_LOGS_PER_PAGE` | `10000` | A page at this known/suspected result cap is incomplete | +| `BASE_HISTORY_MAX_LOGS` | `100000` | Maximum retained events across accepted pages | + +Set the page span and result cap to the limits of your selected provider. A +failed, malformed, capped or unfinished range returns exit 2 when no accounting +violation has been established; it never falls back to nonce sampling or skips +the failed range. An already-observed canonical unledgered movement remains +exit 1 even if a later page fails. Exit 0 requires complete matching history and +successful snapshot state checks. Pending or post-snapshot ledger transactions +are inconclusive, rather than falsely reported absent. Larger histories may +need different explicit budgets; these defaults are not provider capacity or +runtime guarantees. Existing RPC pacing and retry settings still apply. diff --git a/docs/verification.md b/docs/verification.md index 30fdee1..d7c43c1 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -6,73 +6,66 @@ reason. `script/verify_ledger.py` makes that claim falsifiable. ## The chain is the source -History is derived from the chain itself, needing **no API key, no account and +History is read from the chain itself, needing **no API key, no account and nobody's permission**. Any public Base RPC endpoint will serve it. -Reading every block since deployment would be O(chain) — eleven million blocks -to find eleven transactions. Instead it is O(our own activity): +Every block from the token's deployment (41988939) to a fixed head is read in +order, in `eth_getLogs` pages, with nothing skipped. The head is pinned at the +start and re-checked at the end so a reorg cannot turn a later fork's state +into a finding. -1. A nonce is monotonic, so binary-searching `eth_getTransactionCount` over - block height finds every block in which a project wallet sent a - transaction. Nothing hides there: unlike a balance, a nonce never comes - back down, so tokens sent out and returned are still two increments. -2. Logs are read for those blocks only, plus the deployment block for the - mint, which no wallet signed. +That is deliberately the expensive option. It was not always. -Measured against Base mainnet: **about 180 requests, under twenty seconds**, -for the token's entire history. That cost grows with our activity, not with -the chain's. +## Why not a cheaper method -## What a nonce cannot see +Between 2026-10-29 and 2026-11-16 this script used wallet nonces as an index: +a nonce is monotonic, so binary-searching `eth_getTransactionCount` finds every +block in which a project wallet *signed* a transaction, and only those blocks +need their logs read. About 180 requests instead of sixteen thousand. -A third party spending an allowance via `transferFrom` moves tokens without -our wallet signing anything. No nonce increments, and the transfer sits in a -block this derivation never reads. +It was wrong, and [#10](https://github.com/khalidsaidi/sourcemint/issues/10) +is why. A nonce only moves when our wallet signs. A spender using an allowance +moves tokens by `transferFrom` without touching it, so those blocks were never +read. An allowance check was supposed to cover that case, but an allowance is +only evidence while it is live: send tokens out, send them back, revoke the +approval, and the balance nets to zero, the nonce never moved, the allowance +reads zero, and the verifier reported success for history it had never seen. -Two things address it. +Reported by @Chandain, reproduced independently by @gjwang, fixed by @gjwang +in [#11](https://github.com/khalidsaidi/sourcemint/pull/11). Both cases are +permanent offline regressions in `script/test_history_scan.py`. -`balanceOf` and `totalSupply` are reconciled against what the ledger implies, -so an unexplained balance reveals a movement nobody here signed. That catches -the ordinary case but not a round trip, where tokens leave and the same amount -returns and the balance is unchanged. +The lesson is narrower than "nonces are bad": the method sampled, and a sample +cannot support a claim of completeness. Anything cheaper than reading every +block has to prove it is not sampling. -So the verifier also refuses to claim completeness while any allowance is -outstanding. Granting an allowance is itself a transaction a project wallet -signed, so it appears in a block already being read: the same single request -per block returns `Approval` logs alongside `Transfer` logs, at no extra cost. -Every spender ever approved is then checked with `allowance`, and a non-zero -result fails the run with the spender named. +## Three states, not two -That is deliberately strict. A live allowance does not mean anything has gone -wrong — it means this check cannot prove that nothing has, and saying so is -the point. +A single pass/fail flag conflates things that are not the same, which is how +an outage used to downgrade an already-observed violation to "could not +verify". They are now distinct: -## There is no indexer +- An unledgered movement that was actually observed is a **finding**. It stays + a finding whatever happens afterwards — a later page failing, a state read + timing out, the budget running out. +- Coverage that did not complete is **inconclusive**. No verdict is available + about blocks that were not read. +- A ledger entry simply absent from an incomplete range is **neither**. It may + be in a block not yet reached. -There was one, and it is gone. `base.blockscout.com` returns HTTP 403 to -programmatic requests, and no free keyless indexer for Base serves this — -Routescan does not support the chain, BaseScan's v1 API is deprecated, -Etherscan's v2 requires a paid plan for 8453, and Blockscout's MCP server runs -on a shared credit allowance that is routinely exhausted. +## What this costs, and for how long -It was briefly kept as a shortcut. That was a mistake: approvals are only -collected while reading the chain, so a run that took the shortcut skipped the -completeness guard and said nothing about it. A faster path that quietly -checks less is worse than no faster path. +At 1,000 blocks per page — the largest range free keyless endpoints serve — +the token's history is about 16,300 requests and an hour. -If a free keyless indexer reappears it belongs here only if it supplies the -`Approval` logs too. +Base produces roughly 317,000 blocks a day, so that grows by about eight +minutes of scanning every week. Extrapolated, a complete pass passes GitHub +Actions' six-hour job limit within about a year of 2026-11-16. -## Configuration - -| Variable | Default | Used for | -|---|---|---| -| `BASE_RPC_URL` | — | nonces, balances, block height | -| `BASE_LOGS_RPC_URL` | `base.gateway.tenderly.co` | the handful of single-block log reads | - -Two endpoints because no single free one serves both: `mainnet.base.org` -answers nonces and balances but refuses `eth_getLogs` entirely. Both are -public and need no key. +This is a known limit, not a solved problem. When it binds, the options are a +keyed endpoint with larger ranges, or a committed checkpoint of the last +verified block and its hash — which trades "every run re-derives everything" +for "every run trusts the last one", and should not be done quietly. ## Verdicts diff --git a/docs/verify.html b/docs/verify.html index 210a874..ce41b52 100644 --- a/docs/verify.html +++ b/docs/verify.html @@ -137,10 +137,11 @@

What this checked

That proves every claim in the ledger is real. It does not, on its own, prove there is nothing missing from it.

Prove nothing is missing

-

For that, the full history has to be derived from the chain rather than read from the ledger. Your browser can do that too. An account's nonce only ever increases, so searching it over block height finds every block in which a project wallet sent a transaction — nothing can hide there. The logs in those blocks are then read directly.

-

It takes a few hundred requests and under a minute. Same method the weekly check runs.

+

For that the history has to be read from the chain rather than from the ledger — every block since the token was deployed, in order, with nothing skipped.

+

This page used to shortcut that by searching each wallet's nonce for the blocks it had acted in. That was wrong, and Chandain and gjwang showed why: a nonce only moves when our wallet signs something. Someone spending an approved allowance moves tokens without touching it, so those blocks were never read — and if the tokens came back and the approval was revoked, every trace was gone and the page reported success.

+

So it now scans continuously, and tells you exactly how far it got. Base produces blocks faster than a browser can read them, so you will not reach the head in one sitting — the weekly check does the complete pass. Anything this finds is real; what it has not covered, it says it has not covered.

- +

@@ -180,17 +181,49 @@

Doing it yourself instead

const bump = () => { calls++; const el = document.getElementById("calls"); if (el && el.textContent) el.textContent = el.textContent.replace(/\d+ requests/, calls + " requests"); }; -async function rpc(method, params, endpoint = RPC) { - bump(); - const res = await fetch(endpoint, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }) - }); - if (!res.ok) throw new Error(method + ": HTTP " + res.status); - const body = await res.json(); - if (body.error) throw new Error(method + ": " + (body.error.message || "RPC error")); - return body.result; +// Public endpoints throttle a few hundred requests from one browser. A 429 is +// "slow down", not "stop", so retry it and widen the gap for every later call +// — otherwise the next few hundred hit the same wall. +let pace = 40, lastCall = 0; +const sleep = ms => new Promise(r => setTimeout(r, ms)); +const transient = s => s === 429 || s === 500 || s === 502 || s === 503 || s === 504; + +async function rpc(method, params, endpoint = RPC, attempts = 5) { + let last = ""; + for (let i = 0; i < attempts; i++) { + const gap = pace - (Date.now() - lastCall); + if (gap > 0) await sleep(gap); + lastCall = Date.now(); + bump(); + let res; + try { + res = await fetch(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }) + }); + } catch (e) { last = method + ": " + e.message; await sleep(2 ** i * 250); continue; } + + if (!res.ok) { + last = method + ": HTTP " + res.status; + if (!transient(res.status)) throw new Error(last); + pace = Math.min(pace * 2, 800); + await sleep(2 ** i * 250); + continue; + } + const body = await res.json(); + if (body.error) { + const msg = String(body.error.message || "RPC error"); + last = method + ": " + msg; + // a throttle can arrive inside a perfectly successful response + if (!/limit|rate|throttl|too many|capacity/i.test(msg)) throw new Error(last); + pace = Math.min(pace * 2, 800); + await sleep(2 ** i * 250); + continue; + } + return body.result; + } + throw new Error(last || method + ": gave up"); } const short = a => a.slice(0, 6) + "…" + a.slice(-4); @@ -294,56 +327,42 @@

Doing it yourself instead

return problems; } -// ---- tier two: derive the whole history, so nothing can be missing --------- -async function blocksWithTransactions(addr, low, high, onStep) { - const nonceAt = async b => parseInt(await rpc("eth_getTransactionCount", [addr, "0x" + b.toString(16)]), 16); - const found = []; - async function walk(lo, hi, nLo, nHi) { - if (nHi === nLo) return; - if (hi - lo <= 1) { for (let i = 0; i < nHi - nLo; i++) found.push(hi); onStep(); return; } - const mid = Math.floor((lo + hi) / 2), nMid = await nonceAt(mid); - await walk(lo, mid, nLo, nMid); - await walk(mid, hi, nMid, nHi); - } - await walk(low, high, await nonceAt(low), await nonceAt(high)); - return found; -} +// ---- tier two: continuous coverage, honest about how far it got ------------ +// A nonce only moves when our wallet signs. A spender using an allowance moves +// tokens without it, so nonce-based discovery never reads those blocks — the +// bug Chandain and gjwang reported in issue #10. The only honest method is to +// read every block in order and report exactly how far we got. -async function deriveHistory(ledger, progress) { - const head = parseInt(await rpc("eth_blockNumber", []), 16); - const wallets = Object.values(ledger.project_wallets); - const expected = wallets.reduce((n, _) => n, 0); - - const blocks = new Set([DEPLOYMENT_BLOCK]); // the mint, which no wallet signed - let done = 0; - const totals = []; - for (const w of wallets) - totals.push(parseInt(await rpc("eth_getTransactionCount", [w, "0x" + head.toString(16)]), 16)); - const totalTx = totals.reduce((a, b) => a + b, 0); - - for (const w of wallets) { - const bs = await blocksWithTransactions(w, DEPLOYMENT_BLOCK - 1, head, - () => progress(++done / Math.max(totalTx, 1) * 0.85)); - bs.forEach(b => blocks.add(b)); - } +const SPAN = 1000; // the largest range free endpoints serve +const PAGE_BUDGET = 300; // a demonstration a visitor will actually wait for +async function scanHistory(ledger, head, progress) { const token = ledger.token.toLowerCase(); const moves = []; - const sorted = [...blocks].sort((a, b) => a - b); - let read = 0; - for (const b of sorted) { - const logs = await rpc("eth_getLogs", - [{ address: token, fromBlock: "0x" + b.toString(16), toBlock: "0x" + b.toString(16) }], LOGS_RPC); + let from = DEPLOYMENT_BLOCK, covered = from - 1, pages = 0; + + while (from <= head && pages < PAGE_BUDGET) { + const to = Math.min(from + SPAN - 1, head); + const logs = await rpc("eth_getLogs", [{ + address: token, + fromBlock: "0x" + from.toString(16), + toBlock: "0x" + to.toString(16), + topics: [TRANSFER_TOPIC] + }], LOGS_RPC); for (const l of logs) { if (l.removed) continue; // dropped by a reorg - if (!l.topics.length || l.topics[0].toLowerCase() !== TRANSFER_TOPIC || l.topics.length !== 3) continue; - const from = "0x" + l.topics[1].slice(26), to = "0x" + l.topics[2].slice(26); - const ours = from.toLowerCase() === ZERO || walletName(ledger, from); - if (ours) moves.push({ block: b, from, to, wei: BigInt(l.data), tx: l.transactionHash }); + if (l.topics.length !== 3) continue; + const f = "0x" + l.topics[1].slice(26), t = "0x" + l.topics[2].slice(26); + // every movement out of a project wallet, whoever signed for it + if (f.toLowerCase() === ZERO || walletName(ledger, f)) + moves.push({ block: parseInt(l.blockNumber, 16), from: f, to: t, + wei: BigInt(l.data), tx: l.transactionHash }); } - progress(0.85 + (++read / sorted.length) * 0.15); + covered = to; from = to + 1; pages++; + progress((covered - DEPLOYMENT_BLOCK) / Math.max(head - DEPLOYMENT_BLOCK, 1), + covered, head); } - return moves; + return { moves, covered, complete: covered >= head }; } // ---- wiring ---------------------------------------------------------------- @@ -389,14 +408,19 @@

Doing it yourself instead

const btn = this, bar = document.getElementById("bar"), fill = document.getElementById("barfill"); const callsEl = document.getElementById("calls"), table = document.getElementById("derived"); const drows = document.getElementById("drows"); - btn.disabled = true; btn.textContent = "Deriving…"; + btn.disabled = true; btn.textContent = "Scanning…"; bar.style.display = "block"; - callsEl.textContent = "Searching block height by nonce — " + calls + " requests so far."; + callsEl.textContent = "Reading every block in order — " + calls + " requests so far."; let ledger; try { ledger = await (await fetch(LEDGER_URL, { cache: "no-store" })).json(); - const moves = await deriveHistory(ledger, p => { fill.style.width = Math.min(p * 100, 100) + "%"; }); + const head = parseInt(await rpc("eth_blockNumber", []), 16); + const { moves, covered, complete } = await scanHistory(ledger, head, (p, at, h) => { + fill.style.width = Math.min(p * 100, 100) + "%"; + callsEl.textContent = "block " + at.toLocaleString() + " of " + h.toLocaleString() + + " — " + calls + " requests"; + }); // every movement the chain shows, against every movement the ledger claims const claimed = new Map(); @@ -424,23 +448,30 @@

Doing it yourself instead

drows.appendChild(tr); } table.style.display = ""; - fill.style.width = "100%"; - if (unaccounted === 0) { + const span = (covered - DEPLOYMENT_BLOCK + 1).toLocaleString(); + const whole = (head - DEPLOYMENT_BLOCK + 1).toLocaleString(); + const pct = ((covered - DEPLOYMENT_BLOCK + 1) / (head - DEPLOYMENT_BLOCK + 1) * 100).toFixed(1); + + // An unledgered movement that was actually seen is a finding either way. + // Not having seen one, over a range we did not finish, is not a result. + if (unaccounted > 0) { + verdict("bad", unaccounted + " movement(s) are not in the ledger", + "The chain shows project funds moving with no recorded reason. That stands regardless of how much of the history was covered."); + } else if (complete) { verdict("ok", "Nothing is missing either", - moves.length + " project-wallet movements exist on Base since deployment, and every one is in the ledger. Derived in your browser from " + calls + " requests, with no indexer."); - callsEl.textContent = calls + " requests — full history derived from the chain."; + moves.length + " project-wallet movements exist on Base since deployment, and every one is in the ledger. Read block by block in your browser, " + calls + " requests, no indexer."); } else { - verdict("bad", unaccounted + " movement(s) are not in the ledger", - "The chain shows project funds moving with no recorded reason."); - callsEl.textContent = calls + " requests."; + verdict("warn", "Covered " + pct + "% — nothing wrong so far", + "Read blocks up to " + covered.toLocaleString() + " of " + head.toLocaleString() + " (" + span + " of " + whole + "). Everything in that range is ledgered. The rest is simply unread, which is not the same as clean — the weekly check does the complete pass."); } + callsEl.textContent = calls + " requests — covered " + span + " of " + whole + " blocks."; } catch (err) { - verdict("warn", "Couldn't finish deriving the history", + verdict("warn", "Couldn't finish the scan", err.message + " — a provider limit or network problem, not an accounting finding."); callsEl.textContent = calls + " requests before it stopped."; } finally { - btn.textContent = "Derive the full history"; + btn.textContent = "Scan the history"; btn.disabled = false; } }); diff --git a/script/chain_history.py b/script/chain_history.py index a530457..870aed3 100644 --- a/script/chain_history.py +++ b/script/chain_history.py @@ -1,21 +1,11 @@ #!/usr/bin/env python3 -"""Derive a project wallet's complete movement history from the chain. +"""Find blocks containing transactions signed by a wallet. -Scanning every block since deployment is O(chain). This is O(our activity): - - * A nonce is monotonic. Binary-searching `eth_getTransactionCount` over - block height finds every block in which a wallet sent a transaction. - Nothing can hide there — unlike a balance, a nonce never comes back down, - so a transfer out and straight back is still two nonce increments. - * Logs are then read for those blocks alone. Eleven blocks, not eleven - million. - -What a nonce cannot see is a movement the wallet did not sign: a third party -spending an allowance via `transferFrom`. That case is the exact complement — -it changes the balance without touching the nonce — so `unexplained_balance` -below catches it with two reads. Together they leave nothing uncovered. - -Stdlib only. +Nonce changes do not discover token movements initiated by third parties. +A transferFrom roundtrip can restore both balance and allowance, so these +helpers are not a complete token-history source. The verifier scans token +Transfer logs continuously instead. unexplained_balance compares state only; +a zero difference does not establish movement-history completeness. """ @@ -45,10 +35,9 @@ def walk(lo, hi, nonce_lo, nonce_hi): def unexplained_balance(expected_wei, actual_wei): - """What `transferFrom` would look like: a balance the ledger cannot explain. + """Return the expected-minus-observed balance difference in wei. - A nonce search accounts for everything the wallet sent. If the balance is - still short, tokens left without the wallet signing — someone spent an - allowance. Returns the shortfall in wei; zero means fully explained. + A zero state difference cannot prove that no intermediate movements + occurred. Use continuous token logs when verifying transfer history. """ return expected_wei - actual_wei diff --git a/script/test_allowance_outage.py b/script/test_allowance_outage.py new file mode 100644 index 0000000..a26c587 --- /dev/null +++ b/script/test_allowance_outage.py @@ -0,0 +1,61 @@ +"""Coverage and state outages must preserve confirmed accounting findings. + +Supersedes the old allowance-read gate now that Transfer history is scanned. +""" +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +import verify_ledger as verifier +from test_allowances import ledger, transfers + + +class AccountingOutages(unittest.TestCase): + def run_verifier(self, observed=None, balance_error=None, complete=True): + history = verifier.TransferHistory( + transfers() if observed is None else observed, + verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=complete, canonical=True, + error="later page unavailable" if not complete else "") + output = io.StringIO() + with (patch.object(verifier, "LEDGER_PATH") as ledger_path, + patch.object(verifier, "fetch_transfers", return_value=history), + patch.object(verifier, "verify_balances_total_supply", side_effect=balance_error), + patch.object(verifier, "rpc_call", side_effect=lambda payload, endpoint=None: { + "jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}}), + contextlib.redirect_stdout(output)): + ledger_path.read_text.return_value = json.dumps(ledger()) + code = verifier.main() + return code, output.getvalue() + + def test_observed_unaccounted_transfer_survives_state_outage(self): + observed = transfers() + observed.append(dict(observed[-1], transaction_hash="0x" + "22" * 32)) + code, output = self.run_verifier(observed, + verifier.SourceUnavailable("state unavailable")) + self.assertEqual(code, 1) + self.assertIn("UNACCOUNTED", output) + + def test_observed_balance_mismatch_remains_a_violation(self): + code, output = self.run_verifier( + balance_error=verifier.AccountingViolation("balance mismatch")) + self.assertEqual(code, 1) + self.assertIn("BALANCE UNEXPLAINED", output) + + def test_no_observed_violation_with_state_outage_is_inconclusive(self): + code, output = self.run_verifier( + balance_error=verifier.SourceUnavailable("state unavailable")) + self.assertEqual(code, 2) + self.assertIn("COULD NOT VERIFY", output) + + def test_missing_entry_in_partial_history_is_not_a_finding(self): + code, output = self.run_verifier(observed=transfers()[:1], complete=False) + self.assertEqual(code, 2) + self.assertIn("COULD NOT VERIFY", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/script/test_allowances.py b/script/test_allowances.py index 77691ad..2dfd1c6 100644 --- a/script/test_allowances.py +++ b/script/test_allowances.py @@ -1,9 +1,7 @@ -"""An outstanding allowance means the history cannot be called complete. +"""Complete Transfer history covers delegated transfers independently of allowance. -A spender holding an allowance can move tokens via `transferFrom`. That bumps -no nonce on our side and lands in a block the derivation never reads, so it is -invisible to every other check here. The verifier must not print "every -movement is accounted for" while one is live. +These state fixtures retain nonzero and revoked allowances to ensure that +future spending authority does not become a false historical violation. """ import contextlib import io @@ -48,17 +46,21 @@ def run(allowance): def rpc(payload, endpoint=None): if payload["method"] == "eth_blockNumber": return {"jsonrpc": "2.0", "id": 1, "result": "0x1"} + if payload["method"] == "eth_getBlockByNumber": + return {"jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}} data = payload["params"][0]["data"] if data.startswith(verifier.ALLOWANCE_SELECTOR): - value = allowance + raise AssertionError("complete history must not query current allowance") elif data.startswith("0x18160ddd"): value = state[1] else: value = state[0].get("0x" + data[-40:], 0) return {"jsonrpc": "2.0", "id": 1, "result": "0x%064x" % value} - verifier.LIVE_SPENDERS[:] = [PERMIT2] - with (patch.object(verifier, "fetch_transfers", return_value=transfers()), + with (patch.object(verifier, "fetch_transfers", return_value=verifier.TransferHistory( + transfers(), verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=True, canonical=True)), patch.object(verifier, "rpc_call", side_effect=rpc), patch.object(Path, "read_text", return_value=json.dumps(led))): buf = io.StringIO() @@ -68,28 +70,22 @@ def rpc(payload, endpoint=None): class Allowances(unittest.TestCase): - def tearDown(self): - verifier.LIVE_SPENDERS.clear() - - def test_a_live_allowance_prevents_claiming_completeness(self): + def test_a_live_allowance_does_not_invalidate_complete_history(self): code, out = run(UNLIMITED) - self.assertEqual(code, 1, "cannot claim completeness while tokens can leave unseen") - self.assertIn("OUTSTANDING ALLOWANCES", out) - self.assertIn("RESULT: FAIL", out) + self.assertEqual(code, 0) + self.assertNotIn("OUTSTANDING ALLOWANCES", out) - def test_an_unlimited_allowance_is_named_as_such(self): - self.assertIn("unlimited", run(UNLIMITED)[1]) + def test_an_unlimited_allowance_does_not_require_an_extra_history_gate(self): + self.assertIn("RESULT: OK", run(UNLIMITED)[1]) - def test_a_small_allowance_still_counts(self): - code, out = run(5 * 10**18) - self.assertEqual(code, 1) - self.assertIn("5 SRCMNT", out) + def test_a_small_allowance_is_not_a_missing_transfer(self): + self.assertEqual(run(5 * 10**18)[0], 0) def test_a_revoked_allowance_does_not_block_the_verdict(self): code, out = run(0) self.assertEqual(code, 0) self.assertNotIn("OUTSTANDING", out) - self.assertIn("accounted for", out) + self.assertIn("RESULT: OK", out) if __name__ == "__main__": diff --git a/script/test_budget_reserve.py b/script/test_budget_reserve.py new file mode 100644 index 0000000..50760c1 --- /dev/null +++ b/script/test_budget_reserve.py @@ -0,0 +1,60 @@ +"""The mandatory head re-check must stay affordable in time, not just requests. + +A full scan runs for an hour, so it will spend the clock down to the deadline. +If only requests are reserved, the canonical re-check that confirms the fixed +head is unaffordable the moment the scan ends, and a complete, clean history +still reports itself unverified. Found by running the real scan against Base +mainnet, where it returned: + + COULD NOT VERIFY: fixed-head history was not confirmed canonical. + RPC time budget exhausted before retry +""" +import unittest +from unittest.mock import patch + +import verify_ledger as v + + +class TimeReserve(unittest.TestCase): + def budget(self, now): + with patch.object(v.time, "monotonic", return_value=now): + return v.RpcBudget(requests=100, seconds=100) + + def test_scan_stops_early_so_the_recheck_can_still_run(self): + b = self.budget(0.0) + b.reserve, b.reserve_seconds = 2, 10.0 + # 95s in: past the scan's effective deadline (100 - 10), not the real one + with patch.object(v.time, "monotonic", return_value=95.0): + with self.assertRaises(v.SourceUnavailable): + b.check() + # the scan is over; both reserves lift together + b.reserve, b.reserve_seconds = 0, 0.0 + b.check() # the head re-check is still affordable + + def test_requests_are_reserved_the_same_way(self): + b = self.budget(0.0) + b.reserve = 2 + b.used = 98 + with patch.object(v.time, "monotonic", return_value=1.0): + with self.assertRaises(v.SourceUnavailable): + b.check() + b.reserve = 0 + b.check() + + def test_a_retry_pause_respects_the_reserve(self): + b = self.budget(0.0) + b.reserve_seconds = 10.0 + with patch.object(v.time, "monotonic", return_value=85.0): + # 5s remain before the scan deadline; a 6s backoff would eat the reserve + with self.assertRaises(v.SourceUnavailable): + b.pause(6.0) + + def test_the_real_deadline_still_ends_the_run(self): + b = self.budget(0.0) + with patch.object(v.time, "monotonic", return_value=101.0): + with self.assertRaises(v.SourceUnavailable): + b.check() + + +if __name__ == "__main__": + unittest.main() diff --git a/script/test_entries_scope.py b/script/test_entries_scope.py new file mode 100644 index 0000000..8af9c3c --- /dev/null +++ b/script/test_entries_scope.py @@ -0,0 +1,93 @@ +"""The fast per-PR check must be real, and must not imply completeness. + +Proving nothing is missing costs an hour against free endpoints, so a pull +request answers the cheaper question: is every entry a real transfer, and do +the balances add up. Issue #10 happened because a check that could not see +something reported success anyway, so this one has to say what it did not +check, every time. +""" +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +import verify_ledger as v + +TOKEN = "0x914F6b6a0DaD39A3CB932dcf4D1af885C7c08EeB" +A = "0x1af3cBcE941a2AC9D987E853b5385b05c405E16F" +B = "0x7Be179c6Aa4cD1C4B3cDFbeF574c955DABa2c70c" +TX = "0x" + "cd" * 32 + + +def ledger(amount="500"): + return {"token": TOKEN, "chain_id": 8453, + "project_wallets": {"rewards": A}, + "entries": [{"tx": TX, "date": "2026-10-19", "from": A, "to": B, + "amount": amount, "reason": "r"}]} + + +def receipt(to=B, wei=500 * 10**18, status=1): + return {"status": hex(status), "transactionHash": TX, + "blockHash": "0x" + "ab" * 32, "blockNumber": hex(v.DEPLOYMENT_BLOCK + 1), + "logs": [{"address": TOKEN, "topics": [ + v.TRANSFER_TOPIC, "0x" + "0" * 24 + A[2:].lower(), + "0x" + "0" * 24 + to[2:].lower()], "data": hex(wei)}]} + + +class EntriesScope(unittest.TestCase): + def run_it(self, led, rcpt, balances=None): + out = io.StringIO() + def rpc(payload, endpoint=None): + if payload["method"] == "eth_getTransactionReceipt": + return {"jsonrpc": "2.0", "id": 1, "result": rcpt} + raise AssertionError(payload["method"]) + with (patch.object(v, "LEDGER_PATH") as lp, + patch.dict("os.environ", {"BASE_VERIFY_SCOPE": "entries"}), + patch.object(v, "rpc_call", side_effect=rpc), + patch.object(v, "verify_balances_total_supply", + side_effect=balances), + contextlib.redirect_stdout(out)): + lp.read_text.return_value = json.dumps(led) + return v.main(), out.getvalue() + + def test_a_real_entry_passes_and_says_what_it_did_not_check(self): + code, out = self.run_it(ledger(), receipt()) + self.assertEqual(code, 0, out) + self.assertIn("NOT CHECKED", out) + self.assertIn("missing", out) + + def test_a_wrong_amount_is_a_finding(self): + code, out = self.run_it(ledger(amount="9999"), receipt()) + self.assertEqual(code, 1, out) + + def test_a_wrong_recipient_is_a_finding(self): + code, out = self.run_it(ledger(), receipt(to="0x" + "11" * 20)) + self.assertEqual(code, 1, out) + + def test_a_missing_receipt_is_a_finding(self): + code, out = self.run_it(ledger(), None) + self.assertEqual(code, 1, out) + + def test_a_reverted_transaction_is_a_finding(self): + code, out = self.run_it(ledger(), receipt(status=0)) + self.assertEqual(code, 1, out) + + def test_a_balance_mismatch_is_a_finding(self): + code, out = self.run_it(ledger(), receipt(), + balances=v.AccountingViolation("short")) + self.assertEqual(code, 1, out) + + def test_an_unreadable_source_is_inconclusive_not_a_pass(self): + code, out = self.run_it(ledger(), receipt(), + balances=v.SourceUnavailable("down")) + self.assertEqual(code, 2, out) + + def test_full_scope_is_the_default(self): + with patch.dict("os.environ", {}, clear=True): + import os + self.assertEqual(os.environ.get("BASE_VERIFY_SCOPE", "full"), "full") + + +if __name__ == "__main__": + unittest.main() diff --git a/script/test_history_scan.py b/script/test_history_scan.py new file mode 100644 index 0000000..9ba3deb --- /dev/null +++ b/script/test_history_scan.py @@ -0,0 +1,310 @@ +"""Fixed-head history coverage, including the delegated roundtrip in issue #10. + +The roundtrip is the same scenario reported by Chandain in +https://github.com/khalidsaidi/sourcemint/issues/10; all RPC here is offline. +""" +import copy +import contextlib +import io +import json +import os +import unittest +from unittest.mock import patch + +import verify_ledger as v + +TOKEN = '0x' + 'aa' * 20 +ALICE = '0x' + '11' * 20 +BOB = '0x' + '22' * 20 +CHARLIE = '0x' + '33' * 20 +SUPPLY = 100 * 10**18 + + +def block_hash(n): + return '0x' + format(n, '064x') + + +def event(n, sender, receiver, amount, index=0, tx=None): + return {'address': TOKEN, 'blockNumber': hex(n), 'blockHash': block_hash(n), + 'transactionHash': tx or block_hash(1000 + n), + 'logIndex': hex(index), 'transactionIndex': '0x0', 'removed': False, + 'topics': [v.TRANSFER_TOPIC, '0x' + sender[2:].zfill(64), + '0x' + receiver[2:].zfill(64)], + 'data': '0x' + format(amount, '064x')} + + +def ledger(): + return {'token': TOKEN, 'chain_id': 8453, 'project_wallets': {'alice': ALICE}, + 'entries': [{'tx': block_hash(1001), 'from': 'mint', 'to': ALICE, + 'amount': '100', 'reason': 'genesis'}]} + + +class Node: + def __init__(self, head=202, roundtrip=False): + self.head = head + self.events = [event(1, v.ZERO_ADDRESS, ALICE, SUPPLY)] + if roundtrip: + self.events += [event(200, ALICE, CHARLIE, 40 * 10**18), + event(201, CHARLIE, ALICE, 40 * 10**18)] + self.calls = [] + self.fail_start = None + self.changed_head = False + self.wrong_chain = False + self.receipts = {} + self.head_reads = 0 + + def __call__(self, payload, endpoint=None): + self.calls.append((copy.deepcopy(payload), endpoint)) + method, params = payload['method'], payload['params'] + if method == 'eth_chainId': + result = hex(8454 if self.wrong_chain and endpoint else 8453) + elif method == 'eth_getBlockByNumber': + number = self.head if params[0] == 'latest' else int(params[0], 16) + self.head_reads += 1 + h = block_hash(number) + if self.changed_head and self.head_reads >= 3: + h = block_hash(number + 1) + result = {'number': hex(number), 'hash': h} + elif method == 'eth_getLogs': + f = params[0] + start, end = int(f['fromBlock'], 16), int(f['toBlock'], 16) + assert f['address'] == TOKEN and f['topics'] == [v.TRANSFER_TOPIC] + if self.fail_start is not None and start >= self.fail_start: + raise v.SourceUnavailable('page unavailable') + result = copy.deepcopy([e for e in self.events + if start <= int(e['blockNumber'], 16) <= end]) + elif method == 'eth_getTransactionReceipt': + result = self.receipts.get(params[0]) + elif method == 'eth_call': + call, block = params + assert block == {'blockHash': block_hash(self.head), 'requireCanonical': True} + data = call['data'] + assert not data.startswith(v.ALLOWANCE_SELECTOR), 'allowance is not past history' + if data == v.TOTAL_SUPPLY_SELECTOR: + value = SUPPLY + else: + address = '0x' + data[-40:] + value = 0 + seen = set() + for e in self.events: + identity = (e["blockHash"], e["logIndex"]) + if identity in seen: + continue + seen.add(identity) + if int(e['blockNumber'], 16) > self.head: + continue + n = int(e['data'], 16) + if e['topics'][1][-40:] == address[2:]: + value -= n + if e['topics'][2][-40:] == address[2:]: + value += n + result = '0x' + format(value, '064x') + else: + raise AssertionError('unexpected RPC method ' + method) + return {'jsonrpc': '2.0', 'id': 1, 'result': result} + + +def run(node, led=None, **env): + output = io.StringIO() + settings = {'BASE_HISTORY_BLOCK_SPAN': '10000', 'BASE_HISTORY_MAX_REQUESTS': '2048', + 'BASE_HISTORY_TIMEOUT': '300', 'BASE_HISTORY_MAX_LOGS_PER_PAGE': '10000'} + settings.update({k: str(value) for k, value in env.items()}) + with (patch.dict(os.environ, settings), patch.object(v, 'DEPLOYMENT_BLOCK', 1), + patch.object(v, 'LEDGER_PATH') as path, + patch.object(v, 'rpc_call', side_effect=node), contextlib.redirect_stdout(output)): + path.read_text.return_value = json.dumps(led or ledger()) + result = v.main() + return result, output.getvalue() + + +class HistoryScan(unittest.TestCase): + def test_delegated_roundtrip_is_not_hidden_by_restored_balance_or_revocation(self): + node = Node(roundtrip=True) + code, output = run(node) + self.assertEqual(code, 1, output) + self.assertIn('UNACCOUNTED', output) + self.assertFalse(any(x[0]['method'] == 'eth_getTransactionCount' for x in node.calls)) + + def test_complete_matching_history_can_succeed(self): + self.assertEqual(run(Node())[0], 0) + + def test_pages_cover_every_block_once_including_short_final_page(self): + node = Node(head=27) + self.assertEqual(run(node, BASE_HISTORY_BLOCK_SPAN=10)[0], 0) + pages = [x[0]['params'][0] for x in node.calls if x[0]['method'] == 'eth_getLogs'] + self.assertEqual([(int(p['fromBlock'], 16), int(p['toBlock'], 16)) for p in pages], + [(1, 10), (11, 20), (21, 27)]) + + def test_earlier_confirmed_violation_survives_later_page_outage(self): + node = Node(roundtrip=True) + node.fail_start = 201 + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=100) + self.assertEqual(code, 1, output) + self.assertIn('history incomplete', output) + + def test_partial_history_without_observed_violation_is_inconclusive(self): + node = Node() + node.fail_start = 101 + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=100) + self.assertEqual(code, 2, output) + self.assertIn('COULD NOT VERIFY', output) + + def test_exhausted_request_budget_reserves_canonical_checks(self): + node = Node(head=30) + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=10, BASE_HISTORY_MAX_REQUESTS=8) + self.assertEqual(code, 2, output) + self.assertEqual(len(node.calls), 8) + self.assertIn('budget exhausted', output) + + def test_violation_before_budget_exhaustion_remains_a_finding(self): + node = Node(head=30) + node.events += [event(2, ALICE, CHARLIE, 1)] + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=10, BASE_HISTORY_MAX_REQUESTS=8) + self.assertEqual(code, 1, output) + self.assertEqual(len(node.calls), 8) + + def test_head_change_invalidates_unconfirmed_observations(self): + node = Node(roundtrip=True) + node.changed_head = True + self.assertEqual(run(node)[0], 2) + + def test_two_endpoint_chain_mismatch_stops_before_logs(self): + node = Node(); node.wrong_chain = True + self.assertEqual(run(node)[0], 2) + self.assertFalse(any(x[0]['method'] == 'eth_getLogs' for x in node.calls)) + + def test_duplicate_wire_log_is_not_a_second_movement(self): + node = Node(); node.events *= 2 + self.assertEqual(run(node)[0], 0) + + def test_real_duplicate_movements_with_distinct_log_indices_are_counted(self): + node = Node(); tx = block_hash(1002) + node.events += [event(2, ALICE, CHARLIE, 0, 0, tx), event(2, ALICE, CHARLIE, 0, 1, tx)] + led = ledger(); led['entries'] += [{'tx': tx, 'from': ALICE, 'to': CHARLIE, + 'amount': '0', 'reason': 'one recorded event'}] + self.assertEqual(run(node, led)[0], 1) + + def test_malformed_pages_are_not_empty_complete_pages(self): + changes = [lambda e: e.update(removed=True), + lambda e: e.update(address=BOB), + lambda e: e.update(blockHash='0x123'), + lambda e: e.update(data='0x00'), + lambda e: e['topics'].append(block_hash(1)), + lambda e: e['topics'].__setitem__(1, '0x01' + e['topics'][1][4:])] + for change in changes: + with self.subTest(change=changes.index(change)): + node = Node(); change(node.events[0]) + self.assertEqual(run(node)[0], 2) + + def test_conflicting_duplicate_log_is_inconclusive(self): + node = Node(); changed = copy.deepcopy(node.events[0]); changed['data'] = block_hash(1) + node.events.append(changed) + self.assertEqual(run(node)[0], 2) + + def test_known_result_cap_cannot_mark_a_page_complete(self): + self.assertEqual(run(Node(), BASE_HISTORY_MAX_LOGS_PER_PAGE=1)[0], 2) + + def test_case_normalization_preserves_amount_above_float_precision(self): + node = Node() + node.events[0]['topics'] = ['0x' + x[2:].upper() for x in node.events[0]['topics']] + node.events[0]['transactionHash'] = '0x' + node.events[0]['transactionHash'][2:].upper() + self.assertEqual(run(node)[0], 0) + + def test_future_or_pending_ledger_transaction_is_not_a_false_violation(self): + for receipt in [None, {'blockNumber': hex(203), 'blockHash': block_hash(203), + 'transactionHash': block_hash(9000)}]: + with self.subTest(receipt=receipt): + node = Node(); tx = block_hash(9000); node.receipts[tx] = receipt + led = ledger(); led['entries'].append({'tx': tx, 'from': ALICE, + 'to': BOB, 'amount': '0', 'reason': 'outside snapshot'}) + self.assertEqual(run(node, led)[0], 2) + + def test_complete_absent_entry_with_covered_canonical_receipt_is_a_finding(self): + node = Node(); tx = block_hash(9000) + node.receipts[tx] = {'blockNumber': '0x2', 'blockHash': block_hash(2), + 'transactionHash': tx} + led = ledger(); led['entries'].append({'tx': tx, 'from': ALICE, 'to': BOB, + 'amount': '0', 'reason': 'not in receipt logs'}) + self.assertEqual(run(node, led)[0], 1) + + def test_invalid_configuration_is_inconclusive_without_rpc(self): + for key, value in [('BASE_HISTORY_BLOCK_SPAN', '0'), ('BASE_HISTORY_TIMEOUT', 'nan'), + ('BASE_HISTORY_MAX_REQUESTS', '-1')]: + with self.subTest(key=key): + node = Node(); self.assertEqual(run(node, **{key: value})[0], 2) + self.assertEqual(node.calls, []) + + + def test_wrong_header_height_is_not_accepted_as_the_requested_snapshot(self): + class WrongHeight(Node): + def __call__(self, payload, endpoint=None): + result = super().__call__(payload, endpoint) + if payload['method'] == 'eth_getBlockByNumber' and payload['params'][0] != 'latest': + result['result']['number'] = hex(int(result['result']['number'], 16) + 1) + return result + self.assertEqual(run(WrongHeight())[0], 2) + + def test_one_confirmed_absent_entry_survives_another_pending_entry(self): + node = Node(); led = ledger() + for i in [9000, 9001]: + tx = block_hash(i) + led['entries'].append({'tx': tx, 'from': ALICE, 'to': BOB, + 'amount': '0', 'reason': 'claimed event'}) + node.receipts[block_hash(9000)] = { + 'transactionHash': block_hash(9000), 'blockNumber': '0x2', 'blockHash': block_hash(2)} + self.assertEqual(run(node, led)[0], 1) + + def test_total_retained_log_budget_leaves_coverage_incomplete(self): + node = Node(roundtrip=True) + self.assertEqual(run(node, BASE_HISTORY_MAX_LOGS=1)[0], 2) + + def test_a_list_without_coverage_metadata_cannot_claim_verified(self): + with patch.object(v, 'fetch_transfers', return_value=[]): + self.assertEqual(run(Node())[0], 2) + + +class WireBudget(unittest.TestCase): + def setUp(self): + v._last_call[0] = 0 + v._pace[0] = 0 + + def test_every_actual_retry_consumes_one_budget_unit(self): + class Response: + def __enter__(self): return self + def __exit__(self, *a): return False + def read(self, size): + return json.dumps({'jsonrpc': '2.0', 'id': 1, + 'error': {'code': -32005, 'message': 'rate limit'}}).encode() + budget = v.RpcBudget(requests=3) + with (patch.object(v, 'urlopen', return_value=Response()) as opened, + patch.object(v.time, 'sleep')): + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {'jsonrpc': '2.0', 'id': 1, + 'method': 'eth_blockNumber', 'params': []}, 'https://invalid.test') + self.assertEqual(opened.call_count, 3) + self.assertEqual(budget.used, 3) + + def test_response_body_read_is_bounded(self): + class Response: + def __enter__(self): return self + def __exit__(self, *a): return False + def read(self, size): + self.size = size + return b'x' * size + response = Response(); budget = v.RpcBudget(response_bytes=8) + with patch.object(v, 'urlopen', return_value=response): + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {}, 'https://invalid.test') + self.assertEqual(response.size, 9) + self.assertEqual(budget.used, 1) + + def test_expired_deadline_does_not_send_a_request(self): + budget = v.RpcBudget(); budget.deadline = 0 + with patch.object(v, 'urlopen') as opened: + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {}, 'https://invalid.test') + opened.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/script/test_ledger_matching.py b/script/test_ledger_matching.py index b65adaa..add764b 100644 --- a/script/test_ledger_matching.py +++ b/script/test_ledger_matching.py @@ -61,9 +61,17 @@ def balances_for(led): """ state = verifier.calculate_expected_state(led) - def rpc(payload): + def rpc(payload, endpoint=None): if payload["method"] == "eth_blockNumber": return {"jsonrpc": "2.0", "id": 1, "result": "0x1"} + if payload["method"] == "eth_getTransactionReceipt": + return {"jsonrpc": "2.0", "id": 1, "result": { + "transactionHash": payload["params"][0], + "blockNumber": hex(verifier.DEPLOYMENT_BLOCK), + "blockHash": "0x" + "ab" * 32}} + if payload["method"] == "eth_getBlockByNumber": + return {"jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}} data = payload["params"][0]["data"] if data.startswith("0x18160ddd"): value = state[1] @@ -76,7 +84,9 @@ def rpc(payload): def run(transfers, led=None): led = led or ledger() - with (patch.object(verifier, "fetch_transfers", return_value=transfers), + with (patch.object(verifier, "fetch_transfers", return_value=verifier.TransferHistory( + transfers, verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=True, canonical=True)), patch.object(verifier, "rpc_call", side_effect=balances_for(led)), patch.object(Path, "read_text", return_value=json.dumps(led))): buf = io.StringIO() diff --git a/script/test_snapshot_consistency.py b/script/test_snapshot_consistency.py new file mode 100644 index 0000000..491119f --- /dev/null +++ b/script/test_snapshot_consistency.py @@ -0,0 +1,118 @@ +"""Counterexamples from the independent review of the fixed-head scan.""" +import copy +import unittest + +import verify_ledger as v +import test_history_scan as h + + +def entry(tx, amount='40', sender=h.ALICE, receiver=h.BOB): + return {'tx': tx, 'from': sender, 'to': receiver, 'amount': amount, 'reason': 'snapshot boundary'} + + +class SnapshotConsistency(unittest.TestCase): + def test_pending_and_future_nonzero_transfers_are_inconclusive(self): + for receipt in [None, {'transactionHash': h.block_hash(9000), + 'blockNumber': hex(203), 'blockHash': h.block_hash(203)}]: + with self.subTest(receipt=receipt): + node = h.Node(); led = h.ledger(); led['entries'].append(entry(h.block_hash(9000))) + node.receipts[h.block_hash(9000)] = receipt + self.assertEqual(h.run(node, led)[0], 2) + + def test_pending_mint_or_burn_does_not_make_a_supply_finding(self): + for sender, receiver in [('mint', h.ALICE), (h.ALICE, v.ZERO_ADDRESS)]: + with self.subTest(sender=sender): + node = h.Node(); led = h.ledger() + led['entries'].append(entry(h.block_hash(9000), sender=sender, receiver=receiver)) + self.assertEqual(h.run(node, led)[0], 2) + + def test_independent_transfer_finding_survives_pending_nonzero_entry(self): + node = h.Node(); node.events.append(h.event(2, h.ALICE, h.CHARLIE, 0)) + led = h.ledger(); led['entries'].append(entry(h.block_hash(9000))) + self.assertEqual(h.run(node, led)[0], 1) + + def test_independent_missing_finding_survives_pending_nonzero_entry(self): + node = h.Node(); led = h.ledger() + led['entries'] += [entry(h.block_hash(9000), amount='0'), entry(h.block_hash(9001))] + node.receipts[h.block_hash(9000)] = {'transactionHash': h.block_hash(9000), + 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 1) + + def test_malformed_receipt_transaction_hash_is_inconclusive_not_traceback(self): + for value in [None, 123, [], '0x12', h.block_hash(7)]: + with self.subTest(value=value): + node = h.Node(); led = h.ledger(); led['entries'].append(entry(h.block_hash(9000), amount='0')) + node.receipts[h.block_hash(9000)] = {'transactionHash': value, + 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 2) + + def test_same_log_identity_at_two_heights_invalidates_history(self): + node = h.Node(); other = h.event(101, h.ALICE, h.BOB, 0) + other['blockHash'] = h.block_hash(1); node.events.append(other) + self.assertEqual(h.run(node, BASE_HISTORY_BLOCK_SPAN=100)[0], 2) + + def test_different_hashes_at_one_height_invalidate_history(self): + node = h.Node(); other = h.event(1, h.ALICE, h.BOB, 0, index=1) + other['blockHash'] = h.block_hash(500); node.events.append(other) + self.assertEqual(h.run(node)[0], 2) + + def test_a_log_at_head_must_have_the_frozen_head_hash(self): + node = h.Node(); other = h.event(202, h.ALICE, h.BOB, 0) + other['blockHash'] = h.block_hash(500); node.events.append(other) + self.assertEqual(h.run(node)[0], 2) + + def test_reorg_during_receipt_check_does_not_create_a_false_finding(self): + class ReorgNode(h.Node): + changed = False + def __call__(self, payload, endpoint=None): + if payload['method'] == 'eth_getTransactionReceipt': + self.changed = True + if self.changed and payload['method'] == 'eth_call': + raise v.SourceUnavailable('old snapshot is not canonical') + result = super().__call__(payload, endpoint) + if self.changed and payload['method'] == 'eth_getBlockByNumber': + result['result']['hash'] = h.block_hash(int(result['result']['number'], 16) + 500) + return result + node = ReorgNode(); tx = h.block_hash(9000) + node.receipts[tx] = {'transactionHash': tx, 'blockNumber': '0x2', 'blockHash': h.block_hash(502)} + led = h.ledger(); led['entries'].append(entry(tx, amount='0')) + self.assertEqual(h.run(node, led)[0], 2) + + def test_stable_head_missing_finding_survives_a_state_only_outage(self): + class StateOutage(h.Node): + def __call__(self, payload, endpoint=None): + if payload['method'] == 'eth_call': + raise v.SourceUnavailable('state read unavailable') + return super().__call__(payload, endpoint) + node = StateOutage(); tx = h.block_hash(9000) + node.receipts[tx] = {'transactionHash': tx, 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + led = h.ledger(); led['entries'].append(entry(tx, amount='0')) + self.assertEqual(h.run(node, led)[0], 1) + + def test_natural_head_growth_does_not_move_the_snapshot(self): + class Growing(h.Node): + advanced = False + def __call__(self, payload, endpoint=None): + result = super().__call__(payload, endpoint) + if payload['method'] == 'eth_getLogs': + self.advanced = True + if self.advanced and payload['method'] == 'eth_getBlockByNumber' and payload['params'][0] == 'latest': + result['result'] = {'number': hex(self.head + 1), 'hash': h.block_hash(self.head + 1)} + return result + self.assertEqual(h.run(Growing())[0], 0) + + + def test_same_hash_at_different_heights_with_distinct_indices_is_rejected(self): + node = h.Node(); other = h.event(101, h.ALICE, h.BOB, 0, index=1) + other['blockHash'] = h.block_hash(1); node.events.append(other) + self.assertEqual(h.run(node, BASE_HISTORY_BLOCK_SPAN=100)[0], 2) + + def test_bad_receipt_does_not_downgrade_an_independent_transfer_finding(self): + node = h.Node(); node.events.append(h.event(2, h.ALICE, h.CHARLIE, 0)) + tx = h.block_hash(9000); led = h.ledger(); led['entries'].append(entry(tx)) + node.receipts[tx] = {'transactionHash': None, 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/script/verify_ledger.py b/script/verify_ledger.py index 19c6058..80f278e 100644 --- a/script/verify_ledger.py +++ b/script/verify_ledger.py @@ -10,14 +10,15 @@ 0 verified: every project-wallet movement is accounted for 1 ACCOUNTING VIOLATION: an unaccounted movement, or a ledger entry the chain does not show - 2 COULD NOT VERIFY: no data source was reachable (an infrastructure - outage, not a finding about the ledger) + 2 COULD NOT VERIFY: coverage, canonical identity or a required read + could not be established within the configured budgets -The 1/2 split matters. The public indexers this reads are intermittently -unavailable, and an outage must never be reported as an accounting -violation. Deferring is safe: each run re-checks the token's entire -transfer history rather than a delta, so nothing can slip through a missed -run - the next successful run still sees it. +A run scans this token's Transfer logs continuously from DEPLOYMENT_BLOCK +through a fixed head. Success trusts complete RPC responses; it is not an +independent receipt proof. A later failed page preserves canonical observations, +while missing coverage cannot prove a ledger entry false. State reads use the +same canonical block hash. Budgets can make a run inconclusive; no nonce-based +fallback is used. Run: python3 script/verify_ledger.py """ @@ -25,6 +26,9 @@ import json import sys from collections import Counter +from contextvars import ContextVar +from dataclasses import dataclass +import math import time import re import os @@ -35,31 +39,15 @@ from pathlib import Path if __package__: - from .chain_history import blocks_with_transactions from .decode_transfer_fixture import decode_transfer else: - from chain_history import blocks_with_transactions from decode_transfer_fixture import decode_transfer LEDGER_PATH = Path(__file__).resolve().parent.parent / "rewards" / "ledger.json" -# The chain is the source of truth. Transfer history is derived from it with -# no API key, no account and nobody's permission — see derive_transfers below. -# The indexers here answer the same question in one request instead of ~190, -# so they are tried first as a shortcut. Losing them costs seconds, not the -# ability to verify. +# Scan the configured token from its deployment through one fixed head. +# These endpoints must serve the ledger's chain; both are checked before use. DEPLOYMENT_BLOCK = 41988939 - -# Spenders a project wallet has ever granted an allowance to. A live allowance -# means tokens can leave via transferFrom: no nonce increment, and in a block -# the derivation never reads. The verifier must not claim completeness while -# one is outstanding, so it checks each and says so plainly. -LIVE_SPENDERS = [] - -# No single free Base endpoint serves both halves of the derivation, so there -# are two. Nonces and balances come from BASE_RPC_URL; mainnet.base.org is -# fine for those and refuses eth_getLogs entirely. The handful of single-block -# log reads go to BASE_LOGS_RPC_URL. Both are public and need no key. LOGS_RPC_URL = os.environ.get( "BASE_LOGS_RPC_URL", "https://base.gateway.tenderly.co") DECIMALS = 10**18 @@ -112,14 +100,11 @@ def is_hex_bytes(value, size): ) -# Deriving history makes a couple of hundred calls back to back. Free -# endpoints throttle that, so leave a gap between them; the whole run still -# finishes in well under a minute. +# Retain pacing/retry settings, but every wire attempt during verification +# shares the history's request, elapsed-time and response-size budgets. RPC_MIN_INTERVAL = float(os.environ.get("BASE_RPC_MIN_INTERVAL", "0.05")) _last_call = [0.0] _pace = [RPC_MIN_INTERVAL] # grows if the endpoint pushes back -# ~180 sequential calls: a shared CI egress IP gets throttled where a laptop -# does not, so give up only after a few minutes rather than a few seconds. RPC_ATTEMPTS = int(os.environ.get("BASE_RPC_ATTEMPTS", "6")) @@ -148,9 +133,12 @@ def _is_transient(error: dict) -> bool: def rpc_call(payload: dict, endpoint: str = None) -> dict: + budget = _active_budget.get() + if budget is not None: + budget.check() wait = _pace[0] - (time.monotonic() - _last_call[0]) if wait > 0: - time.sleep(wait) + budget.pause(wait) if budget is not None else time.sleep(wait) _last_call[0] = time.monotonic() endpoint = endpoint or os.environ.get("BASE_RPC_URL") @@ -169,8 +157,8 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: method="POST", ) - # Deriving history is ~180 sequential calls. One blip must not discard the - # run, and throttling arrives in three different shapes: an HTTP status, a + # Retry transient transport failures within the shared run budget. + # Throttling arrives in three different shapes: an HTTP status, a # dropped connection, or — the one that is easy to miss — a perfectly # successful HTTP 200 whose JSON-RPC body carries an error. All three are # transient and all three are retried. @@ -178,15 +166,22 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: cause = None # keep the original exception so the traceback survives for attempt in range(RPC_ATTEMPTS): try: - with urlopen(http_request, timeout=30) as resp: - response = json.load(resp) + if budget is not None: + budget.attempt() + timeout = min(30, budget.deadline - time.monotonic()) if budget else 30 + with urlopen(http_request, timeout=timeout) as resp: + cap = budget.response_bytes if budget else 8 * 1024**2 + body = resp.read(cap + 1) + if len(body) > cap: + raise SourceUnavailable("RPC response exceeds byte budget") + response = json.loads(body) except HTTPError as e: if e.code not in (429, 500, 502, 503, 504): raise SourceUnavailable(f"RPC HTTP {e.code}") from e last, cause = f"RPC HTTP {e.code}", e if e.code == 429: _pace[0] = min(max(_pace[0], 0.05) * 2, 1.0) - except (URLError, TimeoutError, json.JSONDecodeError) as e: + except (URLError, TimeoutError, json.JSONDecodeError, UnicodeError) as e: last, cause = f"RPC transport/JSON failure: {type(e).__name__}", e else: error = response.get("error") if isinstance(response, dict) else None @@ -196,87 +191,254 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: _pace[0] = min(max(_pace[0], 0.05) * 2, 1.0) if attempt < RPC_ATTEMPTS - 1: - time.sleep(2 ** (attempt + 1)) + delay = 2 ** (attempt + 1) + budget.pause(delay) if budget is not None else time.sleep(delay) raise SourceUnavailable(last or "unknown RPC error") from cause -def derive_transfers(token: str, wallets): - """Every SRCMNT movement involving a project wallet, derived from the chain. +_WIRE_RPC_CALL = rpc_call + + +@dataclass +class TransferHistory: + transfers: list + head: int + head_hash: str + covered_to: int + complete: bool = False + canonical: bool = False + error: str = "" + budget: object = None + + @property + def block_ref(self): + return {"blockHash": self.head_hash, "requireCanonical": True} + + +_active_budget = ContextVar("ledger_rpc_budget", default=None) + + +class RpcBudget: + def __init__(self, requests=2048, seconds=300, response_bytes=8 * 1024**2): + if type(requests) is not int or requests <= 0: + raise SourceUnavailable("request budget must be a positive integer") + if not math.isfinite(seconds) or seconds <= 0: + raise SourceUnavailable("time budget must be finite and positive") + if type(response_bytes) is not int or response_bytes <= 0: + raise SourceUnavailable("response budget must be a positive integer") + self.limit = requests + self.used = 0 + self.reserve = 0 + # Seconds held back the same way `reserve` holds back requests. The + # canonical head re-check is mandatory and costs time as well as + # requests, so a scan that spends the clock down to the deadline + # leaves it unaffordable and the run can never confirm. Both reserves + # are lifted together once the scan is done. + self.reserve_seconds = 0.0 + self.deadline = time.monotonic() + seconds + self.response_bytes = response_bytes + + def remaining(self): + return self.deadline - self.reserve_seconds - time.monotonic() + + def check(self): + if self.used >= self.limit - self.reserve: + raise SourceUnavailable("RPC request budget exhausted") + if self.remaining() <= 0: + raise SourceUnavailable("RPC time budget exhausted") + + def attempt(self): + self.check() + self.used += 1 + + def pause(self, seconds): + if seconds >= self.remaining(): + raise SourceUnavailable("RPC time budget exhausted before retry") + time.sleep(seconds) + + def call(self, rpc, payload, endpoint=None): + self.check() + before = self.used + token = _active_budget.set(self) + try: + # Tests and custom adapters may implement one logical request + # without entering rpc_call's wire-attempt loop. + return rpc(payload) if endpoint is None else rpc(payload, endpoint=endpoint) + finally: + _active_budget.reset(token) + if self.used == before and rpc is not _WIRE_RPC_CALL: + self.attempt() + - Reading all 11M+ blocks since deployment to find a handful of transactions - is O(chain). This is O(our own activity): +def _quantity(value): + if not isinstance(value, str) or re.fullmatch( + r"0x(?:0|[1-9a-fA-F][0-9a-fA-F]*)", value) is None: + raise SourceUnavailable("invalid RPC quantity") + return int(value, 16) - 1. A nonce is monotonic, so binary-searching `eth_getTransactionCount` - over block height finds every block in which a wallet sent a - transaction. Nothing hides there — unlike a balance, a nonce never - comes back down, so tokens sent out and returned are still two - separate increments. - 2. Logs are read only for those blocks, plus the deployment block for - the mint, which no wallet signed. - A movement the wallet did not sign — a third party spending an allowance - via `transferFrom` — leaves the nonce untouched. That case is the exact - complement and is caught by the balance reconciliation in main(). +def _request(budget, method, params, endpoint=None): + return _rpc_result(budget.call(rpc_call, { + "jsonrpc": "2.0", "id": 1, "method": method, "params": params, + }, endpoint), method) + + +def _header(budget, number, endpoint=None): + result = _request(budget, "eth_getBlockByNumber", [number, False], endpoint) + if number != "latest" and _quantity(result["number"]) != _quantity(number): + raise SourceUnavailable("RPC returned the wrong block height") + return result - Needs no API key, no account and no indexer. + +class HistoryConflict(SourceUnavailable): + """RPC event metadata contradicts this snapshot or earlier observations.""" + + +def _history_page(logs, token, start, end, head, head_hash): + page = {} + for raw in logs: + if not isinstance(raw, dict) or raw.get("removed", False) is not False: + raise SourceUnavailable("invalid or removed log") + if not is_hex_bytes(raw.get("address"), 20) or raw["address"].lower() != token.lower(): + raise SourceUnavailable("log belongs to another token") + number = _quantity(raw.get("blockNumber")) + index = _quantity(raw.get("logIndex")) + tx_index = _quantity(raw.get("transactionIndex")) + if not start <= number <= end or not is_hex_bytes(raw.get("blockHash"), 32): + raise SourceUnavailable("log is outside the requested range or has no block hash") + topics = raw.get("topics") + if not isinstance(topics, list) or len(topics) != 3 or any( + not is_hex_bytes(t, 32) for t in topics): + raise SourceUnavailable("invalid ERC20 Transfer topics") + if topics[0].lower() != TRANSFER_TOPIC: + raise SourceUnavailable("unexpected event signature") + if any(int(t[2:26], 16) for t in topics[1:]): + raise SourceUnavailable("nonzero address padding in Transfer topic") + normalized = dict(raw, topics=[t.lower() for t in topics]) + try: + decoded = decode_transfer(normalized) + except (ValueError, TypeError, KeyError) as exc: + raise SourceUnavailable(f"invalid Transfer: {exc}") from exc + decoded["transaction_hash"] = decoded["transaction_hash"].lower() + decoded["from"]["hash"] = decoded["from"]["hash"].lower() + decoded["to"]["hash"] = decoded["to"]["hash"].lower() + identity = (raw["blockHash"].lower(), index) + if number == head and raw["blockHash"].lower() != head_hash: + raise HistoryConflict("log at the fixed head has a different hash") + value = (number, tx_index, index, decoded) + if identity in page and page[identity] != value: + raise HistoryConflict("conflicting copies of one log") + page[identity] = value + return page + + +def derive_transfers(token: str, wallets, chain_id=8453): + """Continuous token-log coverage, never wallet-nonce sampling. + + Successful eth_getLogs responses are trusted RPC evidence, not receipt + proofs. Budgets or bad responses leave explicit incomplete coverage. """ - def nonce_at(address, block): - return int(_rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, - "method": "eth_getTransactionCount", - "params": [address, hex(block)]}), - "eth_getTransactionCount"), 16) - - head = int(_rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, - "method": "eth_blockNumber", "params": []}), - "eth_blockNumber"), 16) + try: + # 1,000 is the largest eth_getLogs range the free, keyless endpoints + # allow; 10,000 is refused outright ("Block range too large: maximum + # 1000 blocks without an access key"), which would leave every run + # covering zero blocks and reporting itself inconclusive forever. + span = int(os.environ.get("BASE_HISTORY_BLOCK_SPAN", "1000")) + log_cap = int(os.environ.get("BASE_HISTORY_MAX_LOGS_PER_PAGE", "10000")) + total_log_cap = int(os.environ.get("BASE_HISTORY_MAX_LOGS", "100000")) + # Covering the token's whole life takes one request per span. That is + # about 16,300 today and grows with the chain, so the budget has to + # exceed it or the run is honest-but-useless. See docs/verification.md + # on why this does not scale indefinitely. + budget = RpcBudget( + int(os.environ.get("BASE_HISTORY_MAX_REQUESTS", "30000")), + float(os.environ.get("BASE_HISTORY_TIMEOUT", "5400")), + int(os.environ.get("BASE_HISTORY_MAX_RESPONSE_BYTES", str(8 * 1024**2)))) + except (TypeError, ValueError, OverflowError) as exc: + raise SourceUnavailable("invalid history budget configuration") from exc + if span <= 0 or log_cap <= 0 or total_log_cap <= 0 or not is_hex_bytes(token, 20): + raise SourceUnavailable("invalid token or history page limits") + if type(chain_id) is not int or chain_id <= 0 or any( + not is_hex_bytes(w, 20) for w in wallets): + raise SourceUnavailable("invalid chain or project wallet") + for endpoint in (None, LOGS_RPC_URL): + if _quantity(_request(budget, "eth_chainId", [], endpoint)) != chain_id: + raise SourceUnavailable("RPC chain ID differs from ledger.chain_id") + header = _header(budget, "latest") + head = _quantity(header["number"]) if head < DEPLOYMENT_BLOCK: raise SourceUnavailable("chain head precedes the deployment block") + history = TransferHistory([], head, header["hash"].lower(), + DEPLOYMENT_BLOCK - 1, budget=budget) + log_header = _header(budget, hex(head), LOGS_RPC_URL) + if log_header["hash"].lower() != history.head_hash: + raise SourceUnavailable("RPC endpoints disagree about the fixed head") + cursor = DEPLOYMENT_BLOCK + seen, number_hash, hash_number = {}, {}, {} + # Reserve both endpoint head checks. Even after a page exhausts its + # request allowance, earlier pages can still be confirmed or rejected. + # The same has to hold for the clock, or a scan that runs to the deadline + # leaves the mandatory re-check unaffordable and nothing can ever confirm. + budget.reserve = 2 + budget.reserve_seconds = float(os.environ.get("BASE_HISTORY_RESERVE_SECONDS", "60")) + try: + while cursor <= head: + end = min(head, cursor + span - 1) + logs = _request(budget, "eth_getLogs", [{ + "address": token, "topics": [TRANSFER_TOPIC], + "fromBlock": hex(cursor), "toBlock": hex(end), + }], LOGS_RPC_URL) + # A provider cap may mean an incomplete array even with HTTP200. + # Do not accept a capped page as complete or advance its cursor. + if len(logs) >= log_cap: + raise SourceUnavailable("log page reached the configured result cap") + page = _history_page(logs, token, cursor, end, head, history.head_hash) + staged_numbers, staged_hashes = dict(number_hash), dict(hash_number) + added = [] + for identity, value in page.items(): + block_hash, _ = identity + number = value[0] + if ((number in staged_numbers and staged_numbers[number] != block_hash) + or (block_hash in staged_hashes and staged_hashes[block_hash] != number)): + raise HistoryConflict("conflicting block number/hash in history") + if identity in seen and seen[identity] != value: + raise HistoryConflict("log identity changed between pages") + staged_numbers[number] = block_hash + staged_hashes[block_hash] = number + if identity not in seen: + added.append(value) + if len(seen) + len(added) > total_log_cap: + raise SourceUnavailable("history log-memory budget exhausted") + seen.update(page) + number_hash, hash_number = staged_numbers, staged_hashes + history.transfers.extend(value[3] for value in sorted(added, + key=lambda value: value[:3])) + history.covered_to = end + cursor = end + 1 + history.complete = True + except HistoryConflict as exc: + # A self-contradictory provider invalidates this run's observations; + # do not preserve a finding produced by the contradictory records. + history.error = str(exc) + return history + except SourceUnavailable as exc: + history.error = str(exc) + finally: + budget.reserve = 0 + budget.reserve_seconds = 0.0 + try: + for endpoint in (None, LOGS_RPC_URL): + current = _header(budget, hex(head), endpoint) + if current["hash"].lower() != history.head_hash: + raise SourceUnavailable("fixed-head hash changed during history scan") + history.canonical = True + except SourceUnavailable as exc: + history.complete = False + history.error = str(exc) + return history - blocks = {DEPLOYMENT_BLOCK} # the mint itself - for address in sorted(wallets): - blocks.update(blocks_with_transactions( - address, DEPLOYMENT_BLOCK - 1, head, nonce_at)) - - transfers = [] - spenders = set() - for block in sorted(blocks): - # One unfiltered read per block gets Transfers and Approvals together; - # filtering by topic would cost a second request for no benefit. - logs = _rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, "method": "eth_getLogs", - "params": [{"address": token, - "fromBlock": hex(block), - "toBlock": hex(block)}]}, - endpoint=LOGS_RPC_URL), - "eth_getLogs") - for log in logs: - if log.get("removed"): - continue # dropped by a reorg; not part of history - topics = log.get("topics") or [] - if not topics: - continue - if topics[0].lower() == TRANSFER_TOPIC.lower(): - transfers.append(decode_transfer(log)) - elif topics[0].lower() == APPROVAL_TOPIC.lower() and len(topics) == 3: - owner = "0x" + topics[1][-40:] - if owner.lower() in {w.lower() for w in wallets}: - spenders.add("0x" + topics[2][-40:]) - return transfers, spenders - - -def fetch_transfers(token: str, wallets): - """The complete transfer history, derived from the chain. - - There is one source and it is the chain. An indexer would answer faster, - but every one of them is somebody's service to withdraw — and a shortcut - that silently skips checks when it works is worse than no shortcut. If a - free, keyless indexer for Base reappears, it belongs here only if it - supplies the approvals too. - """ - transfers, spenders = derive_transfers(token, wallets) - LIVE_SPENDERS.extend(spenders) - return transfers + +def fetch_transfers(token: str, wallets, chain_id=8453): + return derive_transfers(token, wallets, chain_id) def _srcmnt(wei, signed=False): @@ -349,30 +511,33 @@ def _rpc_result(response, method): if not isinstance(result, list): raise SourceUnavailable(f"{method}: expected a list of logs") - elif method == "eth_blockNumber": + elif method in ("eth_blockNumber", "eth_chainId"): if ( not isinstance(result, str) or re.fullmatch(r"0x(?:0|[1-9a-fA-F][0-9a-fA-F]*)", result) is None ): raise SourceUnavailable(f"{method}: invalid block number") + elif method == "eth_getBlockByNumber": + if not isinstance(result, dict) or not is_hex_bytes(result.get("hash"), 32): + raise SourceUnavailable("invalid block header") + _quantity(result.get("number")) + elif method == "eth_getTransactionReceipt": + if result is not None and not isinstance(result, dict): + raise SourceUnavailable("invalid transaction receipt") else: raise SourceUnavailable(f"Unsupported method: {method}") return result -def verify_balances_total_supply(ledger, token, rpc): +def verify_balances_total_supply(ledger, token, rpc, block=None): wallets = ledger["project_wallets"] expected_state = calculate_expected_state(ledger) - response_block = rpc({ - "jsonrpc": "2.0", - "method": "eth_blockNumber", - "params": [], - "id": 1 - }) - block = _rpc_result(response_block, "eth_blockNumber") + if block is None: + block = _rpc_result(rpc({"jsonrpc": "2.0", "method": "eth_blockNumber", + "params": [], "id": 1}), "eth_blockNumber") for name, address in wallets.items(): address = address.lower() @@ -426,10 +591,83 @@ def verify_balances_total_supply(ledger, token, rpc): +def verify_entries_only(ledger, token) -> int: + """Confirm every ledger entry is real, without claiming the ledger is whole. + + Proving nothing is *missing* means reading every block since deployment — + about 16,300 requests and over an hour against free endpoints, which is + not a per-pull-request cost. This answers the cheaper question instead: + does each entry name a real, successful transfer of that amount to that + address, and do the balances and total supply still add up? + + It deliberately cannot detect an unledgered movement. Issue #10 was caused + by a check that could not see something and reported success anyway, so + this one names what it did not check, every time, and the full scan still + runs on main and on the weekly schedule. + """ + wallets = {a.lower() for a in ledger["project_wallets"].values()} + bad = [] + for e in ledger["entries"]: + tx = e["tx"].lower() + try: + receipt = _rpc_result(rpc_call({"jsonrpc": "2.0", "id": 1, + "method": "eth_getTransactionReceipt", "params": [tx]}), + "eth_getTransactionReceipt") + except SourceUnavailable as exc: + print(f"COULD NOT VERIFY: receipt for {tx} unreadable ({exc}).") + return 2 + if receipt is None: + bad.append(f" {tx}: no receipt on chain") + continue + if _quantity(receipt.get("status")) != 1: + bad.append(f" {tx}: transaction failed on chain") + continue + frm = ZERO_ADDRESS if e["from"] == "mint" else e["from"].lower() + want = (frm, e["to"].lower(), amount_to_wei(e["amount"])) + found = False + for log in receipt.get("logs") or []: + if (log.get("address", "").lower() != token.lower() + or len(log.get("topics") or []) != 3 + or log["topics"][0].lower() != TRANSFER_TOPIC): + continue + got = ("0x" + log["topics"][1][-40:].lower(), + "0x" + log["topics"][2][-40:].lower(), + int(log["data"], 16)) + if got == want: + found = True + break + if not found: + bad.append(f" {tx}: no matching SRCMNT transfer of {e['amount']} to {e['to']}") + + print(f"ledger entries confirmed by receipt: {len(ledger['entries']) - len(bad)}" + f"/{len(ledger['entries'])}") + if bad: + print("\nLEDGER entries not confirmed on-chain:") + print("\n".join(bad)) + print("\nRESULT: FAIL — a ledger entry does not match the chain.") + return 1 + try: + verify_balances_total_supply(ledger, token, rpc_call) + except AccountingViolation as exc: + print(f"\nBALANCE UNEXPLAINED: {exc}") + print("RESULT: FAIL — balances do not match what the ledger implies.") + return 1 + except SourceUnavailable as exc: + print(f"COULD NOT VERIFY: balance state unreadable ({exc}).") + return 2 + print("balances and total supply reconcile with the ledger") + print("\nRESULT: ENTRIES OK — every entry is real and the balances add up.") + print("NOT CHECKED: whether anything is missing from the ledger. That needs" + " the full history scan, which runs on main and weekly.") + return 0 + + def main() -> int: ledger = json.loads(LEDGER_PATH.read_text()) token = ledger["token"] wallets = {a.lower() for a in ledger["project_wallets"].values()} + if os.environ.get("BASE_VERIFY_SCOPE", "full") == "entries": + return verify_entries_only(ledger, token) try: # A movement is (tx, from, to, amount). `from` is part of the identity: # without it an entry can name the wrong source wallet and still match. @@ -451,7 +689,10 @@ def main() -> int: checked = 0 try: - transfers = fetch_transfers(token, wallets) + history = fetch_transfers(token, wallets, ledger["chain_id"]) + if not isinstance(history, TransferHistory): + raise SourceUnavailable("history source did not provide coverage metadata") + transfers = history.transfers except SourceUnavailable as e: print("COULD NOT VERIFY: no transfer history could be obtained.") print(f" tried: {e}") @@ -459,6 +700,11 @@ def main() -> int: print(" The next successful run re-checks the full history.") return 2 + if not history.canonical: + print("COULD NOT VERIFY: fixed-head history was not confirmed canonical.") + print(f" {history.error}") + return 2 + for t in transfers: frm = t["from"]["hash"].lower() to = t["to"]["hash"].lower() @@ -501,57 +747,98 @@ def main() -> int: print("\nLEDGER entries not found on-chain:") print("\n".join(missing_onchain)) - # Everything above checks movements we signed. A third party spending an - # allowance moves tokens without touching our nonce and without appearing - # in a block we looked at — the exact complement. A balance that the - # ledger cannot explain is what that looks like. - try: - verify_balances_total_supply(ledger, token, rpc_call) - except AccountingViolation as e: - ok = False - print("\nBALANCE UNEXPLAINED (a movement we did not sign):") - print(f" {e}") - except SourceUnavailable as e: - # A violation already found is a finding; an outage must never - # downgrade it to "could not verify". - if not ok: - print(f"\nnote: balances unchecked ({e})") - else: - print("COULD NOT VERIFY: transfers matched, but balances could " - f"not be read to rule out a movement we did not sign ({e}).") - return 2 + print(f"history coverage: {DEPLOYMENT_BLOCK}..{history.covered_to} " + f"of fixed head {history.head} ({history.head_hash})") + if not history.complete: + print(f"history incomplete: {history.error}") + if unaccounted: + print("\nRESULT: FAIL — confirmed unaccounted movement in covered history.") + return 1 + print("COULD NOT VERIFY: unobserved entries are not findings in partial history.") + return 2 - # An outstanding allowance lets tokens leave without our wallet signing, - # in a block this derivation never reads. Claiming completeness while one - # is live would be claiming more than was checked. - outstanding = [] - for spender in sorted(set(LIVE_SPENDERS)): - for name, wallet in ledger["project_wallets"].items(): + # An absent entry can refer to a pending or post-snapshot transaction. + # Only a canonical receipt inside this covered interval makes absence a + # finding; null, future or unavailable receipts leave it inconclusive. + unresolved = False + missing_finding = False + def state_rpc(payload): + return history.budget.call(rpc_call, payload) if history.budget else rpc_call(payload) + + if history.budget: + history.budget.reserve = 2 # final fixed-head checks cannot be spent on state/retries + if missing_onchain: + for tx in {key[0] for key in entries if entries[key] > observed[key]}: try: - raw = _rpc_result(rpc_call({ - "jsonrpc": "2.0", "id": 1, "method": "eth_call", - "params": [{"to": token, - "data": ALLOWANCE_SELECTOR - + wallet[2:].lower().zfill(64) - + spender[2:].lower().zfill(64)}, - "latest"]}), "allowance") - except SourceUnavailable as e: - print(f"\nCOULD NOT VERIFY: allowance for {spender} unreadable ({e}).") - return 2 - if int(raw, 16): - outstanding.append((name, spender, int(raw, 16))) - - if outstanding: - ok = False - print("\nOUTSTANDING ALLOWANCES (tokens can leave without us signing):") - for name, spender, amount in outstanding: - shown = "unlimited" if amount > 2**255 else f"{amount // DECIMALS:,}" - print(f" {name} -> {spender} : {shown} SRCMNT") - print(" While these stand, transfers made by the spender do not appear") - print(" in any block this check reads. Revoke them or the history is") - print(" not provably complete.") - - print("\nRESULT:", "OK — every project-wallet movement is accounted for." if ok else "FAIL") + receipt = _rpc_result(state_rpc({"jsonrpc": "2.0", "id": 1, + "method": "eth_getTransactionReceipt", "params": [tx]}), + "eth_getTransactionReceipt") + if receipt is None: + unresolved = True + continue + number = _quantity(receipt.get("blockNumber")) + if (not is_hex_bytes(receipt.get("transactionHash"), 32) or + receipt["transactionHash"].lower() != tx or + not is_hex_bytes(receipt.get("blockHash"), 32)): + raise SourceUnavailable("receipt identity is invalid") + if not DEPLOYMENT_BLOCK <= number <= history.head: + unresolved = True + continue + header = _rpc_result(state_rpc({"jsonrpc": "2.0", "id": 1, + "method": "eth_getBlockByNumber", "params": [hex(number), False]}), + "eth_getBlockByNumber") + if (_quantity(header["number"]) != number or + header["hash"].lower() != receipt["blockHash"].lower()): + unresolved = True + else: + missing_finding = True + except SourceUnavailable as exc: + print(f"note: missing-entry receipt unchecked ({exc})") + unresolved = True + ok = not (unaccounted or missing_finding) + + # A pending/future ledger item does not belong to the H-time state. + # Never compare its nonzero amount against H and call that a violation. + state_unavailable = False + if not unresolved: + try: + verify_balances_total_supply(ledger, token, state_rpc, history.block_ref) + except AccountingViolation as exc: + ok = False + print("\nBALANCE UNEXPLAINED:") + print(f" {exc}") + except SourceUnavailable as exc: + state_unavailable = True + print(f"note: snapshot balances unchecked ({exc})") + + # Receipts and state must not introduce findings from a later fork. + # Confirm H again after those reads, including after state errors. + if history.budget: + history.budget.reserve = 0 + try: + for endpoint in (None, LOGS_RPC_URL): + payload = {"jsonrpc": "2.0", "id": 1, + "method": "eth_getBlockByNumber", "params": [hex(history.head), False]} + response = (history.budget.call(rpc_call, payload, endpoint) + if history.budget else rpc_call(payload, endpoint=endpoint)) + current = _rpc_result(response, "eth_getBlockByNumber") + if (_quantity(current["number"]) != history.head or + current["hash"].lower() != history.head_hash): + raise SourceUnavailable("fixed-head identity changed after receipt/state reads") + except SourceUnavailable as exc: + print(f"COULD NOT VERIFY: final snapshot identity unavailable ({exc}).") + return 2 + + if unresolved and not unaccounted and not missing_finding: + print("COULD NOT VERIFY: ledger entries are outside or unresolved at the snapshot.") + return 2 + if state_unavailable and ok: + print("COULD NOT VERIFY: no finding established and snapshot state unreadable.") + return 2 + # Complete Transfer coverage includes transferFrom irrespective of current + # allowance. A live allowance is future authority, not past missing history. + print("\nRESULT:", "OK — ledger matches trusted RPC history at the fixed head." + if ok else "FAIL") return 0 if ok else 1 diff --git a/test/test_verify_ledger.py b/test/test_verify_ledger.py deleted file mode 100644 index 390da73..0000000 --- a/test/test_verify_ledger.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Where the transfer history comes from, and what happens when it doesn't. - -The chain is the source of truth: history is derived from it with no API key -and nobody's permission. Indexers answer the same question in one request -instead of roughly two hundred, so they are a shortcut tried first — never a -dependency. These tests pin that arrangement and the verdicts around it. -""" -import json -import sys -import unittest -from pathlib import Path -from unittest.mock import patch - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "script")) -import verify_ledger as verifier # noqa: E402 - -TOKEN = "0x914F6b6a0DaD39A3CB932dcf4D1af885C7c08EeB" -TREASURY = "0x497d34b1d0790f374B32467093d303533399c819" -WALLETS = {TREASURY.lower()} - - -def rpc_responses(nonce_by_block, logs_by_block, head): - """Answer the three methods the derivation uses.""" - def rpc(payload, endpoint=None): - method = payload["method"] - if method == "eth_blockNumber": - return {"jsonrpc": "2.0", "id": 1, "result": hex(head)} - if method == "eth_getTransactionCount": - block = int(payload["params"][1], 16) - n = max((v for b, v in sorted(nonce_by_block.items()) if b <= block), - default=0) - return {"jsonrpc": "2.0", "id": 1, "result": hex(n)} - if method == "eth_getLogs": - block = int(payload["params"][0]["fromBlock"], 16) - return {"jsonrpc": "2.0", "id": 1, - "result": logs_by_block.get(block, [])} - raise AssertionError(f"unexpected method {method}") - return rpc - - -def log(frm, to, wei): - return {"topics": [verifier.TRANSFER_TOPIC, - "0x" + "0" * 24 + frm[2:].lower(), - "0x" + "0" * 24 + to[2:].lower()], - "data": "0x" + f"{wei:064x}", - "transactionHash": "0x" + "ab" * 32} - - -class OneSource(unittest.TestCase): - """There is one source and it is the chain. - - An indexer would be faster, but it is somebody's service to withdraw, and - the approvals that guard completeness are only collected while reading the - chain. A shortcut that skips a check when it works is worse than none. - """ - - def test_history_always_comes_from_the_chain(self): - with patch.object(verifier, "derive_transfers", - return_value=(["derived"], {"0xspender"})) as derive: - self.assertEqual(verifier.fetch_transfers(TOKEN, WALLETS), ["derived"]) - derive.assert_called_once() - - def test_approvals_found_while_reading_are_kept_for_the_allowance_check(self): - verifier.LIVE_SPENDERS.clear() - with patch.object(verifier, "derive_transfers", - return_value=([], {"0xabc"})): - verifier.fetch_transfers(TOKEN, WALLETS) - self.assertIn("0xabc", verifier.LIVE_SPENDERS) - verifier.LIVE_SPENDERS.clear() - - def test_a_failure_raises_rather_than_returning_nothing(self): - with patch.object(verifier, "derive_transfers", - side_effect=verifier.SourceUnavailable("rpc down")): - with self.assertRaises(verifier.SourceUnavailable): - verifier.fetch_transfers(TOKEN, WALLETS) - - -class Derivation(unittest.TestCase): - def derive(self, nonces, logs, head=None): - head = head or verifier.DEPLOYMENT_BLOCK + 1_000_000 - with patch.object(verifier, "rpc_call", - side_effect=rpc_responses(nonces, logs, head)): - return verifier.derive_transfers(TOKEN, WALLETS)[0] - - def test_finds_a_movement_in_a_block_the_wallet_transacted_in(self): - out = self.derive({verifier.DEPLOYMENT_BLOCK + 500_000: 1}, - {verifier.DEPLOYMENT_BLOCK + 500_000: [log(TREASURY, TOKEN, 5 * 10**18)]}) - self.assertEqual(len(out), 1) - self.assertEqual(out[0]["from"]["hash"].lower(), TREASURY.lower()) - - def test_an_idle_wallet_yields_only_the_deployment_block(self): - self.assertEqual(self.derive({}, {}), []) - - def test_logs_removed_by_a_reorg_are_discarded(self): - entry = log(TREASURY, TOKEN, 5 * 10**18) - entry["removed"] = True - self.assertEqual(self.derive({verifier.DEPLOYMENT_BLOCK + 500_000: 1}, {500_000: [entry]}), []) - - def test_a_head_before_deployment_is_an_outage_not_a_verdict(self): - with patch.object(verifier, "rpc_call", - side_effect=rpc_responses({}, {}, verifier.DEPLOYMENT_BLOCK - 1)): - with self.assertRaises(verifier.SourceUnavailable): - verifier.derive_transfers(TOKEN, WALLETS) - - -if __name__ == "__main__": - unittest.main()