From bf1a25a8d876094f4e71ac6fa1aff2eed3047650 Mon Sep 17 00:00:00 2001 From: gjwang Date: Fri, 6 Nov 2026 06:20:55 +0800 Subject: [PATCH 1/2] fix: preserve observed findings when allowance reads fail Refs #10. --- script/test_allowance_outage.py | 62 +++++++++++++++++++++++++++++++++ script/verify_ledger.py | 7 ++++ 2 files changed, 69 insertions(+) create mode 100644 script/test_allowance_outage.py diff --git a/script/test_allowance_outage.py b/script/test_allowance_outage.py new file mode 100644 index 0000000..5e9507d --- /dev/null +++ b/script/test_allowance_outage.py @@ -0,0 +1,62 @@ +"""An allowance outage must preserve an already-observed accounting finding.""" +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +import verify_ledger as verifier +from test_allowances import PERMIT2, ledger, transfers + + +class AllowanceOutage(unittest.TestCase): + def run_verifier(self, observed=None, balance_error=None): + output = io.StringIO() + with ( + patch.object(verifier, "LIVE_SPENDERS", [PERMIT2]), + patch.object(verifier, "LEDGER_PATH") as ledger_path, + patch.object(verifier, "fetch_transfers", return_value=( + transfers() if observed is None else observed)), + patch.object(verifier, "verify_balances_total_supply", + side_effect=balance_error), + patch.object(verifier, "rpc_call", side_effect=( + verifier.SourceUnavailable("allowance source unavailable"))) as rpc, + contextlib.redirect_stdout(output), + ): + ledger_path.read_text.return_value = json.dumps(ledger()) + code = verifier.main() + self.assertEqual(rpc.call_count, 1) + self.assertTrue(rpc.call_args.args[0]["params"][0]["data"].startswith( + verifier.ALLOWANCE_SELECTOR)) + return code, output.getvalue() + + def test_observed_unaccounted_transfer_remains_a_violation(self): + observed = transfers() + observed.append(dict(observed[-1], transaction_hash="0x" + "22" * 32)) + code, output = self.run_verifier(observed=observed) + self.assertEqual(code, 1) + self.assertIn("UNACCOUNTED on-chain movements", output) + self.assertIn("allowance", output) + self.assertNotIn("COULD NOT VERIFY", output) + + def test_observed_balance_mismatch_remains_a_violation(self): + code, output = self.run_verifier( + balance_error=verifier.AccountingViolation("balance mismatch")) + self.assertEqual(code, 1) + self.assertIn("BALANCE UNEXPLAINED", output) + self.assertNotIn("COULD NOT VERIFY", output) + + def test_no_observed_violation_remains_inconclusive(self): + code, output = self.run_verifier() + self.assertEqual(code, 2) + self.assertIn("COULD NOT VERIFY", output) + + def test_missing_sampled_entry_alone_is_not_an_observed_violation(self): + code, output = self.run_verifier(observed=transfers()[:1]) + self.assertEqual(code, 2) + self.assertIn("LEDGER entries not found on-chain", output) + self.assertIn("COULD NOT VERIFY", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/script/verify_ledger.py b/script/verify_ledger.py index 19c6058..a22a5b2 100644 --- a/script/verify_ledger.py +++ b/script/verify_ledger.py @@ -505,10 +505,12 @@ def main() -> int: # allowance moves tokens without touching our nonce and without appearing # in a block we looked at — the exact complement. A balance that the # ledger cannot explain is what that looks like. + balance_violation = False try: verify_balances_total_supply(ledger, token, rpc_call) except AccountingViolation as e: ok = False + balance_violation = True print("\nBALANCE UNEXPLAINED (a movement we did not sign):") print(f" {e}") except SourceUnavailable as e: @@ -536,6 +538,11 @@ def main() -> int: + spender[2:].lower().zfill(64)}, "latest"]}), "allowance") except SourceUnavailable as e: + # A missing sampled entry is not an observed violation, but + # an actual transfer or balance finding survives an outage. + if unaccounted or balance_violation: + print(f"\nnote: allowance for {spender} unchecked ({e}).") + return 1 print(f"\nCOULD NOT VERIFY: allowance for {spender} unreadable ({e}).") return 2 if int(raw, 16): From 28cbe58a24e2aeff7e91aadde64ac9b2e5bf79ce Mon Sep 17 00:00:00 2001 From: gjwang Date: Fri, 6 Nov 2026 08:44:00 +0800 Subject: [PATCH 2/2] Verify fixed-head token history with bounded RPC coverage [skip ci] Refs #10 --- .github/workflows/ledger.yml | 12 +- README.md | 33 ++ script/chain_history.py | 29 +- script/test_allowance_outage.py | 57 ++- script/test_allowances.py | 40 +-- script/test_history_scan.py | 310 ++++++++++++++++ script/test_ledger_matching.py | 14 +- script/test_snapshot_consistency.py | 118 ++++++ script/verify_ledger.py | 534 +++++++++++++++++++--------- 9 files changed, 894 insertions(+), 253 deletions(-) create mode 100644 script/test_history_scan.py create mode 100644 script/test_snapshot_consistency.py diff --git a/.github/workflows/ledger.yml b/.github/workflows/ledger.yml index 79b146a..d79ed17 100644 --- a/.github/workflows/ledger.yml +++ b/.github/workflows/ledger.yml @@ -31,11 +31,11 @@ jobs: - name: Validate ledger shape and published figures run: | python3 script/ledger_schema.py - python3 -B -m unittest discover -s script -p 'test_ledger_schema.py' + python3 -B -m unittest discover -s script -p 'test_*.py' - name: Verify ledger against Base mainnet env: - # Nonces and balances. mainnet.base.org serves these and refuses - # eth_getLogs, so the log reads go elsewhere. Both public, no key. + # Snapshot headers and state use the primary RPC; token logs use + # the logs RPC. Both endpoints must report the configured chain. BASE_RPC_URL: https://mainnet.base.org BASE_LOGS_RPC_URL: https://base.gateway.tenderly.co # Shared runner egress gets throttled where a laptop does not. @@ -49,9 +49,9 @@ jobs: echo "Ledger verified against Base mainnet." exit 0 elif [ "$code" -eq 2 ]; then - # An indexer outage is not an accounting finding. Each run re-checks - # the full transfer history, so a deferred check loses nothing. - echo "::warning title=Ledger not verified::Transfer data source unreachable; deferring to the next run." + # Incomplete coverage is not an accounting finding. Every run + # starts again from deployment and reports its actual coverage. + echo "::warning title=Ledger not verified::History or snapshot could not be verified within source/resource limits; deferring." exit 0 else echo "::error title=Accounting violation::A project-wallet movement is unaccounted for in rewards/ledger.json." diff --git a/README.md b/README.md index 9cb6a59..dec679b 100644 --- a/README.md +++ b/README.md @@ -55,3 +55,36 @@ Deployment artifacts: - Base Mainnet (8453): `0x914F6b6a0DaD39A3CB932dcf4D1af885C7c08EeB` (GA) - Verified: https://basescan.org/address/0x914f6b6a0dad39a3cb932dcf4d1af885c7c08eeb#code - Sepolia (11155111): `0x8c2B67EA395824Cc069F5e13b1d80E6016751a4A` (testnet) + +## Ledger verification coverage + +`python3 script/verify_ledger.py` scans this token's `Transfer` logs from its +configured deployment block through a fixed head. Both configured RPC endpoints +must report the ledger's chain ID and agree on that head. Balances and total +supply are read using its canonical block hash. This covers delegated +`transferFrom` movements even if balances later return to their original values +and allowances are revoked. Current allowances are not evidence of missing +historical transfers. + +The verifier trusts complete RPC log responses; this is an accounting check, +not an independent cryptographic receipt proof. It keeps the existing scope: +mint events and transfers out of project wallets must match the ledger. + +| Environment setting | Default | Purpose | +| --- | --- | --- | +| `BASE_HISTORY_BLOCK_SPAN` | `10000` | Maximum blocks per log request | +| `BASE_HISTORY_MAX_REQUESTS` | `2048` | Shared request budget, including wire retries and state reads | +| `BASE_HISTORY_TIMEOUT` | `300` | Deadline in seconds, checked around RPC operations and retry delays | +| `BASE_HISTORY_MAX_RESPONSE_BYTES` | `8388608` | Maximum response body per request | +| `BASE_HISTORY_MAX_LOGS_PER_PAGE` | `10000` | A page at this known/suspected result cap is incomplete | +| `BASE_HISTORY_MAX_LOGS` | `100000` | Maximum retained events across accepted pages | + +Set the page span and result cap to the limits of your selected provider. A +failed, malformed, capped or unfinished range returns exit 2 when no accounting +violation has been established; it never falls back to nonce sampling or skips +the failed range. An already-observed canonical unledgered movement remains +exit 1 even if a later page fails. Exit 0 requires complete matching history and +successful snapshot state checks. Pending or post-snapshot ledger transactions +are inconclusive, rather than falsely reported absent. Larger histories may +need different explicit budgets; these defaults are not provider capacity or +runtime guarantees. Existing RPC pacing and retry settings still apply. diff --git a/script/chain_history.py b/script/chain_history.py index a530457..870aed3 100644 --- a/script/chain_history.py +++ b/script/chain_history.py @@ -1,21 +1,11 @@ #!/usr/bin/env python3 -"""Derive a project wallet's complete movement history from the chain. +"""Find blocks containing transactions signed by a wallet. -Scanning every block since deployment is O(chain). This is O(our activity): - - * A nonce is monotonic. Binary-searching `eth_getTransactionCount` over - block height finds every block in which a wallet sent a transaction. - Nothing can hide there — unlike a balance, a nonce never comes back down, - so a transfer out and straight back is still two nonce increments. - * Logs are then read for those blocks alone. Eleven blocks, not eleven - million. - -What a nonce cannot see is a movement the wallet did not sign: a third party -spending an allowance via `transferFrom`. That case is the exact complement — -it changes the balance without touching the nonce — so `unexplained_balance` -below catches it with two reads. Together they leave nothing uncovered. - -Stdlib only. +Nonce changes do not discover token movements initiated by third parties. +A transferFrom roundtrip can restore both balance and allowance, so these +helpers are not a complete token-history source. The verifier scans token +Transfer logs continuously instead. unexplained_balance compares state only; +a zero difference does not establish movement-history completeness. """ @@ -45,10 +35,9 @@ def walk(lo, hi, nonce_lo, nonce_hi): def unexplained_balance(expected_wei, actual_wei): - """What `transferFrom` would look like: a balance the ledger cannot explain. + """Return the expected-minus-observed balance difference in wei. - A nonce search accounts for everything the wallet sent. If the balance is - still short, tokens left without the wallet signing — someone spent an - allowance. Returns the shortfall in wei; zero means fully explained. + A zero state difference cannot prove that no intermediate movements + occurred. Use continuous token logs when verifying transfer history. """ return expected_wei - actual_wei diff --git a/script/test_allowance_outage.py b/script/test_allowance_outage.py index 5e9507d..a26c587 100644 --- a/script/test_allowance_outage.py +++ b/script/test_allowance_outage.py @@ -1,4 +1,7 @@ -"""An allowance outage must preserve an already-observed accounting finding.""" +"""Coverage and state outages must preserve confirmed accounting findings. + +Supersedes the old allowance-read gate now that Transfer history is scanned. +""" import contextlib import io import json @@ -6,55 +9,51 @@ from unittest.mock import patch import verify_ledger as verifier -from test_allowances import PERMIT2, ledger, transfers +from test_allowances import ledger, transfers -class AllowanceOutage(unittest.TestCase): - def run_verifier(self, observed=None, balance_error=None): +class AccountingOutages(unittest.TestCase): + def run_verifier(self, observed=None, balance_error=None, complete=True): + history = verifier.TransferHistory( + transfers() if observed is None else observed, + verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=complete, canonical=True, + error="later page unavailable" if not complete else "") output = io.StringIO() - with ( - patch.object(verifier, "LIVE_SPENDERS", [PERMIT2]), - patch.object(verifier, "LEDGER_PATH") as ledger_path, - patch.object(verifier, "fetch_transfers", return_value=( - transfers() if observed is None else observed)), - patch.object(verifier, "verify_balances_total_supply", - side_effect=balance_error), - patch.object(verifier, "rpc_call", side_effect=( - verifier.SourceUnavailable("allowance source unavailable"))) as rpc, - contextlib.redirect_stdout(output), - ): + with (patch.object(verifier, "LEDGER_PATH") as ledger_path, + patch.object(verifier, "fetch_transfers", return_value=history), + patch.object(verifier, "verify_balances_total_supply", side_effect=balance_error), + patch.object(verifier, "rpc_call", side_effect=lambda payload, endpoint=None: { + "jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}}), + contextlib.redirect_stdout(output)): ledger_path.read_text.return_value = json.dumps(ledger()) code = verifier.main() - self.assertEqual(rpc.call_count, 1) - self.assertTrue(rpc.call_args.args[0]["params"][0]["data"].startswith( - verifier.ALLOWANCE_SELECTOR)) return code, output.getvalue() - def test_observed_unaccounted_transfer_remains_a_violation(self): + def test_observed_unaccounted_transfer_survives_state_outage(self): observed = transfers() observed.append(dict(observed[-1], transaction_hash="0x" + "22" * 32)) - code, output = self.run_verifier(observed=observed) + code, output = self.run_verifier(observed, + verifier.SourceUnavailable("state unavailable")) self.assertEqual(code, 1) - self.assertIn("UNACCOUNTED on-chain movements", output) - self.assertIn("allowance", output) - self.assertNotIn("COULD NOT VERIFY", output) + self.assertIn("UNACCOUNTED", output) def test_observed_balance_mismatch_remains_a_violation(self): code, output = self.run_verifier( balance_error=verifier.AccountingViolation("balance mismatch")) self.assertEqual(code, 1) self.assertIn("BALANCE UNEXPLAINED", output) - self.assertNotIn("COULD NOT VERIFY", output) - def test_no_observed_violation_remains_inconclusive(self): - code, output = self.run_verifier() + def test_no_observed_violation_with_state_outage_is_inconclusive(self): + code, output = self.run_verifier( + balance_error=verifier.SourceUnavailable("state unavailable")) self.assertEqual(code, 2) self.assertIn("COULD NOT VERIFY", output) - def test_missing_sampled_entry_alone_is_not_an_observed_violation(self): - code, output = self.run_verifier(observed=transfers()[:1]) + def test_missing_entry_in_partial_history_is_not_a_finding(self): + code, output = self.run_verifier(observed=transfers()[:1], complete=False) self.assertEqual(code, 2) - self.assertIn("LEDGER entries not found on-chain", output) self.assertIn("COULD NOT VERIFY", output) diff --git a/script/test_allowances.py b/script/test_allowances.py index 77691ad..2dfd1c6 100644 --- a/script/test_allowances.py +++ b/script/test_allowances.py @@ -1,9 +1,7 @@ -"""An outstanding allowance means the history cannot be called complete. +"""Complete Transfer history covers delegated transfers independently of allowance. -A spender holding an allowance can move tokens via `transferFrom`. That bumps -no nonce on our side and lands in a block the derivation never reads, so it is -invisible to every other check here. The verifier must not print "every -movement is accounted for" while one is live. +These state fixtures retain nonzero and revoked allowances to ensure that +future spending authority does not become a false historical violation. """ import contextlib import io @@ -48,17 +46,21 @@ def run(allowance): def rpc(payload, endpoint=None): if payload["method"] == "eth_blockNumber": return {"jsonrpc": "2.0", "id": 1, "result": "0x1"} + if payload["method"] == "eth_getBlockByNumber": + return {"jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}} data = payload["params"][0]["data"] if data.startswith(verifier.ALLOWANCE_SELECTOR): - value = allowance + raise AssertionError("complete history must not query current allowance") elif data.startswith("0x18160ddd"): value = state[1] else: value = state[0].get("0x" + data[-40:], 0) return {"jsonrpc": "2.0", "id": 1, "result": "0x%064x" % value} - verifier.LIVE_SPENDERS[:] = [PERMIT2] - with (patch.object(verifier, "fetch_transfers", return_value=transfers()), + with (patch.object(verifier, "fetch_transfers", return_value=verifier.TransferHistory( + transfers(), verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=True, canonical=True)), patch.object(verifier, "rpc_call", side_effect=rpc), patch.object(Path, "read_text", return_value=json.dumps(led))): buf = io.StringIO() @@ -68,28 +70,22 @@ def rpc(payload, endpoint=None): class Allowances(unittest.TestCase): - def tearDown(self): - verifier.LIVE_SPENDERS.clear() - - def test_a_live_allowance_prevents_claiming_completeness(self): + def test_a_live_allowance_does_not_invalidate_complete_history(self): code, out = run(UNLIMITED) - self.assertEqual(code, 1, "cannot claim completeness while tokens can leave unseen") - self.assertIn("OUTSTANDING ALLOWANCES", out) - self.assertIn("RESULT: FAIL", out) + self.assertEqual(code, 0) + self.assertNotIn("OUTSTANDING ALLOWANCES", out) - def test_an_unlimited_allowance_is_named_as_such(self): - self.assertIn("unlimited", run(UNLIMITED)[1]) + def test_an_unlimited_allowance_does_not_require_an_extra_history_gate(self): + self.assertIn("RESULT: OK", run(UNLIMITED)[1]) - def test_a_small_allowance_still_counts(self): - code, out = run(5 * 10**18) - self.assertEqual(code, 1) - self.assertIn("5 SRCMNT", out) + def test_a_small_allowance_is_not_a_missing_transfer(self): + self.assertEqual(run(5 * 10**18)[0], 0) def test_a_revoked_allowance_does_not_block_the_verdict(self): code, out = run(0) self.assertEqual(code, 0) self.assertNotIn("OUTSTANDING", out) - self.assertIn("accounted for", out) + self.assertIn("RESULT: OK", out) if __name__ == "__main__": diff --git a/script/test_history_scan.py b/script/test_history_scan.py new file mode 100644 index 0000000..9ba3deb --- /dev/null +++ b/script/test_history_scan.py @@ -0,0 +1,310 @@ +"""Fixed-head history coverage, including the delegated roundtrip in issue #10. + +The roundtrip is the same scenario reported by Chandain in +https://github.com/khalidsaidi/sourcemint/issues/10; all RPC here is offline. +""" +import copy +import contextlib +import io +import json +import os +import unittest +from unittest.mock import patch + +import verify_ledger as v + +TOKEN = '0x' + 'aa' * 20 +ALICE = '0x' + '11' * 20 +BOB = '0x' + '22' * 20 +CHARLIE = '0x' + '33' * 20 +SUPPLY = 100 * 10**18 + + +def block_hash(n): + return '0x' + format(n, '064x') + + +def event(n, sender, receiver, amount, index=0, tx=None): + return {'address': TOKEN, 'blockNumber': hex(n), 'blockHash': block_hash(n), + 'transactionHash': tx or block_hash(1000 + n), + 'logIndex': hex(index), 'transactionIndex': '0x0', 'removed': False, + 'topics': [v.TRANSFER_TOPIC, '0x' + sender[2:].zfill(64), + '0x' + receiver[2:].zfill(64)], + 'data': '0x' + format(amount, '064x')} + + +def ledger(): + return {'token': TOKEN, 'chain_id': 8453, 'project_wallets': {'alice': ALICE}, + 'entries': [{'tx': block_hash(1001), 'from': 'mint', 'to': ALICE, + 'amount': '100', 'reason': 'genesis'}]} + + +class Node: + def __init__(self, head=202, roundtrip=False): + self.head = head + self.events = [event(1, v.ZERO_ADDRESS, ALICE, SUPPLY)] + if roundtrip: + self.events += [event(200, ALICE, CHARLIE, 40 * 10**18), + event(201, CHARLIE, ALICE, 40 * 10**18)] + self.calls = [] + self.fail_start = None + self.changed_head = False + self.wrong_chain = False + self.receipts = {} + self.head_reads = 0 + + def __call__(self, payload, endpoint=None): + self.calls.append((copy.deepcopy(payload), endpoint)) + method, params = payload['method'], payload['params'] + if method == 'eth_chainId': + result = hex(8454 if self.wrong_chain and endpoint else 8453) + elif method == 'eth_getBlockByNumber': + number = self.head if params[0] == 'latest' else int(params[0], 16) + self.head_reads += 1 + h = block_hash(number) + if self.changed_head and self.head_reads >= 3: + h = block_hash(number + 1) + result = {'number': hex(number), 'hash': h} + elif method == 'eth_getLogs': + f = params[0] + start, end = int(f['fromBlock'], 16), int(f['toBlock'], 16) + assert f['address'] == TOKEN and f['topics'] == [v.TRANSFER_TOPIC] + if self.fail_start is not None and start >= self.fail_start: + raise v.SourceUnavailable('page unavailable') + result = copy.deepcopy([e for e in self.events + if start <= int(e['blockNumber'], 16) <= end]) + elif method == 'eth_getTransactionReceipt': + result = self.receipts.get(params[0]) + elif method == 'eth_call': + call, block = params + assert block == {'blockHash': block_hash(self.head), 'requireCanonical': True} + data = call['data'] + assert not data.startswith(v.ALLOWANCE_SELECTOR), 'allowance is not past history' + if data == v.TOTAL_SUPPLY_SELECTOR: + value = SUPPLY + else: + address = '0x' + data[-40:] + value = 0 + seen = set() + for e in self.events: + identity = (e["blockHash"], e["logIndex"]) + if identity in seen: + continue + seen.add(identity) + if int(e['blockNumber'], 16) > self.head: + continue + n = int(e['data'], 16) + if e['topics'][1][-40:] == address[2:]: + value -= n + if e['topics'][2][-40:] == address[2:]: + value += n + result = '0x' + format(value, '064x') + else: + raise AssertionError('unexpected RPC method ' + method) + return {'jsonrpc': '2.0', 'id': 1, 'result': result} + + +def run(node, led=None, **env): + output = io.StringIO() + settings = {'BASE_HISTORY_BLOCK_SPAN': '10000', 'BASE_HISTORY_MAX_REQUESTS': '2048', + 'BASE_HISTORY_TIMEOUT': '300', 'BASE_HISTORY_MAX_LOGS_PER_PAGE': '10000'} + settings.update({k: str(value) for k, value in env.items()}) + with (patch.dict(os.environ, settings), patch.object(v, 'DEPLOYMENT_BLOCK', 1), + patch.object(v, 'LEDGER_PATH') as path, + patch.object(v, 'rpc_call', side_effect=node), contextlib.redirect_stdout(output)): + path.read_text.return_value = json.dumps(led or ledger()) + result = v.main() + return result, output.getvalue() + + +class HistoryScan(unittest.TestCase): + def test_delegated_roundtrip_is_not_hidden_by_restored_balance_or_revocation(self): + node = Node(roundtrip=True) + code, output = run(node) + self.assertEqual(code, 1, output) + self.assertIn('UNACCOUNTED', output) + self.assertFalse(any(x[0]['method'] == 'eth_getTransactionCount' for x in node.calls)) + + def test_complete_matching_history_can_succeed(self): + self.assertEqual(run(Node())[0], 0) + + def test_pages_cover_every_block_once_including_short_final_page(self): + node = Node(head=27) + self.assertEqual(run(node, BASE_HISTORY_BLOCK_SPAN=10)[0], 0) + pages = [x[0]['params'][0] for x in node.calls if x[0]['method'] == 'eth_getLogs'] + self.assertEqual([(int(p['fromBlock'], 16), int(p['toBlock'], 16)) for p in pages], + [(1, 10), (11, 20), (21, 27)]) + + def test_earlier_confirmed_violation_survives_later_page_outage(self): + node = Node(roundtrip=True) + node.fail_start = 201 + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=100) + self.assertEqual(code, 1, output) + self.assertIn('history incomplete', output) + + def test_partial_history_without_observed_violation_is_inconclusive(self): + node = Node() + node.fail_start = 101 + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=100) + self.assertEqual(code, 2, output) + self.assertIn('COULD NOT VERIFY', output) + + def test_exhausted_request_budget_reserves_canonical_checks(self): + node = Node(head=30) + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=10, BASE_HISTORY_MAX_REQUESTS=8) + self.assertEqual(code, 2, output) + self.assertEqual(len(node.calls), 8) + self.assertIn('budget exhausted', output) + + def test_violation_before_budget_exhaustion_remains_a_finding(self): + node = Node(head=30) + node.events += [event(2, ALICE, CHARLIE, 1)] + code, output = run(node, BASE_HISTORY_BLOCK_SPAN=10, BASE_HISTORY_MAX_REQUESTS=8) + self.assertEqual(code, 1, output) + self.assertEqual(len(node.calls), 8) + + def test_head_change_invalidates_unconfirmed_observations(self): + node = Node(roundtrip=True) + node.changed_head = True + self.assertEqual(run(node)[0], 2) + + def test_two_endpoint_chain_mismatch_stops_before_logs(self): + node = Node(); node.wrong_chain = True + self.assertEqual(run(node)[0], 2) + self.assertFalse(any(x[0]['method'] == 'eth_getLogs' for x in node.calls)) + + def test_duplicate_wire_log_is_not_a_second_movement(self): + node = Node(); node.events *= 2 + self.assertEqual(run(node)[0], 0) + + def test_real_duplicate_movements_with_distinct_log_indices_are_counted(self): + node = Node(); tx = block_hash(1002) + node.events += [event(2, ALICE, CHARLIE, 0, 0, tx), event(2, ALICE, CHARLIE, 0, 1, tx)] + led = ledger(); led['entries'] += [{'tx': tx, 'from': ALICE, 'to': CHARLIE, + 'amount': '0', 'reason': 'one recorded event'}] + self.assertEqual(run(node, led)[0], 1) + + def test_malformed_pages_are_not_empty_complete_pages(self): + changes = [lambda e: e.update(removed=True), + lambda e: e.update(address=BOB), + lambda e: e.update(blockHash='0x123'), + lambda e: e.update(data='0x00'), + lambda e: e['topics'].append(block_hash(1)), + lambda e: e['topics'].__setitem__(1, '0x01' + e['topics'][1][4:])] + for change in changes: + with self.subTest(change=changes.index(change)): + node = Node(); change(node.events[0]) + self.assertEqual(run(node)[0], 2) + + def test_conflicting_duplicate_log_is_inconclusive(self): + node = Node(); changed = copy.deepcopy(node.events[0]); changed['data'] = block_hash(1) + node.events.append(changed) + self.assertEqual(run(node)[0], 2) + + def test_known_result_cap_cannot_mark_a_page_complete(self): + self.assertEqual(run(Node(), BASE_HISTORY_MAX_LOGS_PER_PAGE=1)[0], 2) + + def test_case_normalization_preserves_amount_above_float_precision(self): + node = Node() + node.events[0]['topics'] = ['0x' + x[2:].upper() for x in node.events[0]['topics']] + node.events[0]['transactionHash'] = '0x' + node.events[0]['transactionHash'][2:].upper() + self.assertEqual(run(node)[0], 0) + + def test_future_or_pending_ledger_transaction_is_not_a_false_violation(self): + for receipt in [None, {'blockNumber': hex(203), 'blockHash': block_hash(203), + 'transactionHash': block_hash(9000)}]: + with self.subTest(receipt=receipt): + node = Node(); tx = block_hash(9000); node.receipts[tx] = receipt + led = ledger(); led['entries'].append({'tx': tx, 'from': ALICE, + 'to': BOB, 'amount': '0', 'reason': 'outside snapshot'}) + self.assertEqual(run(node, led)[0], 2) + + def test_complete_absent_entry_with_covered_canonical_receipt_is_a_finding(self): + node = Node(); tx = block_hash(9000) + node.receipts[tx] = {'blockNumber': '0x2', 'blockHash': block_hash(2), + 'transactionHash': tx} + led = ledger(); led['entries'].append({'tx': tx, 'from': ALICE, 'to': BOB, + 'amount': '0', 'reason': 'not in receipt logs'}) + self.assertEqual(run(node, led)[0], 1) + + def test_invalid_configuration_is_inconclusive_without_rpc(self): + for key, value in [('BASE_HISTORY_BLOCK_SPAN', '0'), ('BASE_HISTORY_TIMEOUT', 'nan'), + ('BASE_HISTORY_MAX_REQUESTS', '-1')]: + with self.subTest(key=key): + node = Node(); self.assertEqual(run(node, **{key: value})[0], 2) + self.assertEqual(node.calls, []) + + + def test_wrong_header_height_is_not_accepted_as_the_requested_snapshot(self): + class WrongHeight(Node): + def __call__(self, payload, endpoint=None): + result = super().__call__(payload, endpoint) + if payload['method'] == 'eth_getBlockByNumber' and payload['params'][0] != 'latest': + result['result']['number'] = hex(int(result['result']['number'], 16) + 1) + return result + self.assertEqual(run(WrongHeight())[0], 2) + + def test_one_confirmed_absent_entry_survives_another_pending_entry(self): + node = Node(); led = ledger() + for i in [9000, 9001]: + tx = block_hash(i) + led['entries'].append({'tx': tx, 'from': ALICE, 'to': BOB, + 'amount': '0', 'reason': 'claimed event'}) + node.receipts[block_hash(9000)] = { + 'transactionHash': block_hash(9000), 'blockNumber': '0x2', 'blockHash': block_hash(2)} + self.assertEqual(run(node, led)[0], 1) + + def test_total_retained_log_budget_leaves_coverage_incomplete(self): + node = Node(roundtrip=True) + self.assertEqual(run(node, BASE_HISTORY_MAX_LOGS=1)[0], 2) + + def test_a_list_without_coverage_metadata_cannot_claim_verified(self): + with patch.object(v, 'fetch_transfers', return_value=[]): + self.assertEqual(run(Node())[0], 2) + + +class WireBudget(unittest.TestCase): + def setUp(self): + v._last_call[0] = 0 + v._pace[0] = 0 + + def test_every_actual_retry_consumes_one_budget_unit(self): + class Response: + def __enter__(self): return self + def __exit__(self, *a): return False + def read(self, size): + return json.dumps({'jsonrpc': '2.0', 'id': 1, + 'error': {'code': -32005, 'message': 'rate limit'}}).encode() + budget = v.RpcBudget(requests=3) + with (patch.object(v, 'urlopen', return_value=Response()) as opened, + patch.object(v.time, 'sleep')): + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {'jsonrpc': '2.0', 'id': 1, + 'method': 'eth_blockNumber', 'params': []}, 'https://invalid.test') + self.assertEqual(opened.call_count, 3) + self.assertEqual(budget.used, 3) + + def test_response_body_read_is_bounded(self): + class Response: + def __enter__(self): return self + def __exit__(self, *a): return False + def read(self, size): + self.size = size + return b'x' * size + response = Response(); budget = v.RpcBudget(response_bytes=8) + with patch.object(v, 'urlopen', return_value=response): + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {}, 'https://invalid.test') + self.assertEqual(response.size, 9) + self.assertEqual(budget.used, 1) + + def test_expired_deadline_does_not_send_a_request(self): + budget = v.RpcBudget(); budget.deadline = 0 + with patch.object(v, 'urlopen') as opened: + with self.assertRaises(v.SourceUnavailable): + budget.call(v.rpc_call, {}, 'https://invalid.test') + opened.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/script/test_ledger_matching.py b/script/test_ledger_matching.py index b65adaa..add764b 100644 --- a/script/test_ledger_matching.py +++ b/script/test_ledger_matching.py @@ -61,9 +61,17 @@ def balances_for(led): """ state = verifier.calculate_expected_state(led) - def rpc(payload): + def rpc(payload, endpoint=None): if payload["method"] == "eth_blockNumber": return {"jsonrpc": "2.0", "id": 1, "result": "0x1"} + if payload["method"] == "eth_getTransactionReceipt": + return {"jsonrpc": "2.0", "id": 1, "result": { + "transactionHash": payload["params"][0], + "blockNumber": hex(verifier.DEPLOYMENT_BLOCK), + "blockHash": "0x" + "ab" * 32}} + if payload["method"] == "eth_getBlockByNumber": + return {"jsonrpc": "2.0", "id": 1, "result": { + "number": payload["params"][0], "hash": "0x" + "ab" * 32}} data = payload["params"][0]["data"] if data.startswith("0x18160ddd"): value = state[1] @@ -76,7 +84,9 @@ def rpc(payload): def run(transfers, led=None): led = led or ledger() - with (patch.object(verifier, "fetch_transfers", return_value=transfers), + with (patch.object(verifier, "fetch_transfers", return_value=verifier.TransferHistory( + transfers, verifier.DEPLOYMENT_BLOCK, "0x" + "ab" * 32, + verifier.DEPLOYMENT_BLOCK, complete=True, canonical=True)), patch.object(verifier, "rpc_call", side_effect=balances_for(led)), patch.object(Path, "read_text", return_value=json.dumps(led))): buf = io.StringIO() diff --git a/script/test_snapshot_consistency.py b/script/test_snapshot_consistency.py new file mode 100644 index 0000000..491119f --- /dev/null +++ b/script/test_snapshot_consistency.py @@ -0,0 +1,118 @@ +"""Counterexamples from the independent review of the fixed-head scan.""" +import copy +import unittest + +import verify_ledger as v +import test_history_scan as h + + +def entry(tx, amount='40', sender=h.ALICE, receiver=h.BOB): + return {'tx': tx, 'from': sender, 'to': receiver, 'amount': amount, 'reason': 'snapshot boundary'} + + +class SnapshotConsistency(unittest.TestCase): + def test_pending_and_future_nonzero_transfers_are_inconclusive(self): + for receipt in [None, {'transactionHash': h.block_hash(9000), + 'blockNumber': hex(203), 'blockHash': h.block_hash(203)}]: + with self.subTest(receipt=receipt): + node = h.Node(); led = h.ledger(); led['entries'].append(entry(h.block_hash(9000))) + node.receipts[h.block_hash(9000)] = receipt + self.assertEqual(h.run(node, led)[0], 2) + + def test_pending_mint_or_burn_does_not_make_a_supply_finding(self): + for sender, receiver in [('mint', h.ALICE), (h.ALICE, v.ZERO_ADDRESS)]: + with self.subTest(sender=sender): + node = h.Node(); led = h.ledger() + led['entries'].append(entry(h.block_hash(9000), sender=sender, receiver=receiver)) + self.assertEqual(h.run(node, led)[0], 2) + + def test_independent_transfer_finding_survives_pending_nonzero_entry(self): + node = h.Node(); node.events.append(h.event(2, h.ALICE, h.CHARLIE, 0)) + led = h.ledger(); led['entries'].append(entry(h.block_hash(9000))) + self.assertEqual(h.run(node, led)[0], 1) + + def test_independent_missing_finding_survives_pending_nonzero_entry(self): + node = h.Node(); led = h.ledger() + led['entries'] += [entry(h.block_hash(9000), amount='0'), entry(h.block_hash(9001))] + node.receipts[h.block_hash(9000)] = {'transactionHash': h.block_hash(9000), + 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 1) + + def test_malformed_receipt_transaction_hash_is_inconclusive_not_traceback(self): + for value in [None, 123, [], '0x12', h.block_hash(7)]: + with self.subTest(value=value): + node = h.Node(); led = h.ledger(); led['entries'].append(entry(h.block_hash(9000), amount='0')) + node.receipts[h.block_hash(9000)] = {'transactionHash': value, + 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 2) + + def test_same_log_identity_at_two_heights_invalidates_history(self): + node = h.Node(); other = h.event(101, h.ALICE, h.BOB, 0) + other['blockHash'] = h.block_hash(1); node.events.append(other) + self.assertEqual(h.run(node, BASE_HISTORY_BLOCK_SPAN=100)[0], 2) + + def test_different_hashes_at_one_height_invalidate_history(self): + node = h.Node(); other = h.event(1, h.ALICE, h.BOB, 0, index=1) + other['blockHash'] = h.block_hash(500); node.events.append(other) + self.assertEqual(h.run(node)[0], 2) + + def test_a_log_at_head_must_have_the_frozen_head_hash(self): + node = h.Node(); other = h.event(202, h.ALICE, h.BOB, 0) + other['blockHash'] = h.block_hash(500); node.events.append(other) + self.assertEqual(h.run(node)[0], 2) + + def test_reorg_during_receipt_check_does_not_create_a_false_finding(self): + class ReorgNode(h.Node): + changed = False + def __call__(self, payload, endpoint=None): + if payload['method'] == 'eth_getTransactionReceipt': + self.changed = True + if self.changed and payload['method'] == 'eth_call': + raise v.SourceUnavailable('old snapshot is not canonical') + result = super().__call__(payload, endpoint) + if self.changed and payload['method'] == 'eth_getBlockByNumber': + result['result']['hash'] = h.block_hash(int(result['result']['number'], 16) + 500) + return result + node = ReorgNode(); tx = h.block_hash(9000) + node.receipts[tx] = {'transactionHash': tx, 'blockNumber': '0x2', 'blockHash': h.block_hash(502)} + led = h.ledger(); led['entries'].append(entry(tx, amount='0')) + self.assertEqual(h.run(node, led)[0], 2) + + def test_stable_head_missing_finding_survives_a_state_only_outage(self): + class StateOutage(h.Node): + def __call__(self, payload, endpoint=None): + if payload['method'] == 'eth_call': + raise v.SourceUnavailable('state read unavailable') + return super().__call__(payload, endpoint) + node = StateOutage(); tx = h.block_hash(9000) + node.receipts[tx] = {'transactionHash': tx, 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + led = h.ledger(); led['entries'].append(entry(tx, amount='0')) + self.assertEqual(h.run(node, led)[0], 1) + + def test_natural_head_growth_does_not_move_the_snapshot(self): + class Growing(h.Node): + advanced = False + def __call__(self, payload, endpoint=None): + result = super().__call__(payload, endpoint) + if payload['method'] == 'eth_getLogs': + self.advanced = True + if self.advanced and payload['method'] == 'eth_getBlockByNumber' and payload['params'][0] == 'latest': + result['result'] = {'number': hex(self.head + 1), 'hash': h.block_hash(self.head + 1)} + return result + self.assertEqual(h.run(Growing())[0], 0) + + + def test_same_hash_at_different_heights_with_distinct_indices_is_rejected(self): + node = h.Node(); other = h.event(101, h.ALICE, h.BOB, 0, index=1) + other['blockHash'] = h.block_hash(1); node.events.append(other) + self.assertEqual(h.run(node, BASE_HISTORY_BLOCK_SPAN=100)[0], 2) + + def test_bad_receipt_does_not_downgrade_an_independent_transfer_finding(self): + node = h.Node(); node.events.append(h.event(2, h.ALICE, h.CHARLIE, 0)) + tx = h.block_hash(9000); led = h.ledger(); led['entries'].append(entry(tx)) + node.receipts[tx] = {'transactionHash': None, 'blockNumber': '0x2', 'blockHash': h.block_hash(2)} + self.assertEqual(h.run(node, led)[0], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/script/verify_ledger.py b/script/verify_ledger.py index a22a5b2..d23ce9f 100644 --- a/script/verify_ledger.py +++ b/script/verify_ledger.py @@ -10,14 +10,15 @@ 0 verified: every project-wallet movement is accounted for 1 ACCOUNTING VIOLATION: an unaccounted movement, or a ledger entry the chain does not show - 2 COULD NOT VERIFY: no data source was reachable (an infrastructure - outage, not a finding about the ledger) + 2 COULD NOT VERIFY: coverage, canonical identity or a required read + could not be established within the configured budgets -The 1/2 split matters. The public indexers this reads are intermittently -unavailable, and an outage must never be reported as an accounting -violation. Deferring is safe: each run re-checks the token's entire -transfer history rather than a delta, so nothing can slip through a missed -run - the next successful run still sees it. +A run scans this token's Transfer logs continuously from DEPLOYMENT_BLOCK +through a fixed head. Success trusts complete RPC responses; it is not an +independent receipt proof. A later failed page preserves canonical observations, +while missing coverage cannot prove a ledger entry false. State reads use the +same canonical block hash. Budgets can make a run inconclusive; no nonce-based +fallback is used. Run: python3 script/verify_ledger.py """ @@ -25,6 +26,9 @@ import json import sys from collections import Counter +from contextvars import ContextVar +from dataclasses import dataclass +import math import time import re import os @@ -35,31 +39,15 @@ from pathlib import Path if __package__: - from .chain_history import blocks_with_transactions from .decode_transfer_fixture import decode_transfer else: - from chain_history import blocks_with_transactions from decode_transfer_fixture import decode_transfer LEDGER_PATH = Path(__file__).resolve().parent.parent / "rewards" / "ledger.json" -# The chain is the source of truth. Transfer history is derived from it with -# no API key, no account and nobody's permission — see derive_transfers below. -# The indexers here answer the same question in one request instead of ~190, -# so they are tried first as a shortcut. Losing them costs seconds, not the -# ability to verify. +# Scan the configured token from its deployment through one fixed head. +# These endpoints must serve the ledger's chain; both are checked before use. DEPLOYMENT_BLOCK = 41988939 - -# Spenders a project wallet has ever granted an allowance to. A live allowance -# means tokens can leave via transferFrom: no nonce increment, and in a block -# the derivation never reads. The verifier must not claim completeness while -# one is outstanding, so it checks each and says so plainly. -LIVE_SPENDERS = [] - -# No single free Base endpoint serves both halves of the derivation, so there -# are two. Nonces and balances come from BASE_RPC_URL; mainnet.base.org is -# fine for those and refuses eth_getLogs entirely. The handful of single-block -# log reads go to BASE_LOGS_RPC_URL. Both are public and need no key. LOGS_RPC_URL = os.environ.get( "BASE_LOGS_RPC_URL", "https://base.gateway.tenderly.co") DECIMALS = 10**18 @@ -112,14 +100,11 @@ def is_hex_bytes(value, size): ) -# Deriving history makes a couple of hundred calls back to back. Free -# endpoints throttle that, so leave a gap between them; the whole run still -# finishes in well under a minute. +# Retain pacing/retry settings, but every wire attempt during verification +# shares the history's request, elapsed-time and response-size budgets. RPC_MIN_INTERVAL = float(os.environ.get("BASE_RPC_MIN_INTERVAL", "0.05")) _last_call = [0.0] _pace = [RPC_MIN_INTERVAL] # grows if the endpoint pushes back -# ~180 sequential calls: a shared CI egress IP gets throttled where a laptop -# does not, so give up only after a few minutes rather than a few seconds. RPC_ATTEMPTS = int(os.environ.get("BASE_RPC_ATTEMPTS", "6")) @@ -148,9 +133,12 @@ def _is_transient(error: dict) -> bool: def rpc_call(payload: dict, endpoint: str = None) -> dict: + budget = _active_budget.get() + if budget is not None: + budget.check() wait = _pace[0] - (time.monotonic() - _last_call[0]) if wait > 0: - time.sleep(wait) + budget.pause(wait) if budget is not None else time.sleep(wait) _last_call[0] = time.monotonic() endpoint = endpoint or os.environ.get("BASE_RPC_URL") @@ -169,8 +157,8 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: method="POST", ) - # Deriving history is ~180 sequential calls. One blip must not discard the - # run, and throttling arrives in three different shapes: an HTTP status, a + # Retry transient transport failures within the shared run budget. + # Throttling arrives in three different shapes: an HTTP status, a # dropped connection, or — the one that is easy to miss — a perfectly # successful HTTP 200 whose JSON-RPC body carries an error. All three are # transient and all three are retried. @@ -178,15 +166,22 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: cause = None # keep the original exception so the traceback survives for attempt in range(RPC_ATTEMPTS): try: - with urlopen(http_request, timeout=30) as resp: - response = json.load(resp) + if budget is not None: + budget.attempt() + timeout = min(30, budget.deadline - time.monotonic()) if budget else 30 + with urlopen(http_request, timeout=timeout) as resp: + cap = budget.response_bytes if budget else 8 * 1024**2 + body = resp.read(cap + 1) + if len(body) > cap: + raise SourceUnavailable("RPC response exceeds byte budget") + response = json.loads(body) except HTTPError as e: if e.code not in (429, 500, 502, 503, 504): raise SourceUnavailable(f"RPC HTTP {e.code}") from e last, cause = f"RPC HTTP {e.code}", e if e.code == 429: _pace[0] = min(max(_pace[0], 0.05) * 2, 1.0) - except (URLError, TimeoutError, json.JSONDecodeError) as e: + except (URLError, TimeoutError, json.JSONDecodeError, UnicodeError) as e: last, cause = f"RPC transport/JSON failure: {type(e).__name__}", e else: error = response.get("error") if isinstance(response, dict) else None @@ -196,87 +191,233 @@ def rpc_call(payload: dict, endpoint: str = None) -> dict: _pace[0] = min(max(_pace[0], 0.05) * 2, 1.0) if attempt < RPC_ATTEMPTS - 1: - time.sleep(2 ** (attempt + 1)) + delay = 2 ** (attempt + 1) + budget.pause(delay) if budget is not None else time.sleep(delay) raise SourceUnavailable(last or "unknown RPC error") from cause -def derive_transfers(token: str, wallets): - """Every SRCMNT movement involving a project wallet, derived from the chain. +_WIRE_RPC_CALL = rpc_call + + +@dataclass +class TransferHistory: + transfers: list + head: int + head_hash: str + covered_to: int + complete: bool = False + canonical: bool = False + error: str = "" + budget: object = None + + @property + def block_ref(self): + return {"blockHash": self.head_hash, "requireCanonical": True} + + +_active_budget = ContextVar("ledger_rpc_budget", default=None) + + +class RpcBudget: + def __init__(self, requests=2048, seconds=300, response_bytes=8 * 1024**2): + if type(requests) is not int or requests <= 0: + raise SourceUnavailable("request budget must be a positive integer") + if not math.isfinite(seconds) or seconds <= 0: + raise SourceUnavailable("time budget must be finite and positive") + if type(response_bytes) is not int or response_bytes <= 0: + raise SourceUnavailable("response budget must be a positive integer") + self.limit = requests + self.used = 0 + self.reserve = 0 + self.deadline = time.monotonic() + seconds + self.response_bytes = response_bytes + + def check(self): + if self.used >= self.limit - self.reserve: + raise SourceUnavailable("RPC request budget exhausted") + if time.monotonic() >= self.deadline: + raise SourceUnavailable("RPC time budget exhausted") + + def attempt(self): + self.check() + self.used += 1 + + def pause(self, seconds): + if seconds >= self.deadline - time.monotonic(): + raise SourceUnavailable("RPC time budget exhausted before retry") + time.sleep(seconds) + + def call(self, rpc, payload, endpoint=None): + self.check() + before = self.used + token = _active_budget.set(self) + try: + # Tests and custom adapters may implement one logical request + # without entering rpc_call's wire-attempt loop. + return rpc(payload) if endpoint is None else rpc(payload, endpoint=endpoint) + finally: + _active_budget.reset(token) + if self.used == before and rpc is not _WIRE_RPC_CALL: + self.attempt() + - Reading all 11M+ blocks since deployment to find a handful of transactions - is O(chain). This is O(our own activity): +def _quantity(value): + if not isinstance(value, str) or re.fullmatch( + r"0x(?:0|[1-9a-fA-F][0-9a-fA-F]*)", value) is None: + raise SourceUnavailable("invalid RPC quantity") + return int(value, 16) - 1. A nonce is monotonic, so binary-searching `eth_getTransactionCount` - over block height finds every block in which a wallet sent a - transaction. Nothing hides there — unlike a balance, a nonce never - comes back down, so tokens sent out and returned are still two - separate increments. - 2. Logs are read only for those blocks, plus the deployment block for - the mint, which no wallet signed. - A movement the wallet did not sign — a third party spending an allowance - via `transferFrom` — leaves the nonce untouched. That case is the exact - complement and is caught by the balance reconciliation in main(). +def _request(budget, method, params, endpoint=None): + return _rpc_result(budget.call(rpc_call, { + "jsonrpc": "2.0", "id": 1, "method": method, "params": params, + }, endpoint), method) + + +def _header(budget, number, endpoint=None): + result = _request(budget, "eth_getBlockByNumber", [number, False], endpoint) + if number != "latest" and _quantity(result["number"]) != _quantity(number): + raise SourceUnavailable("RPC returned the wrong block height") + return result + - Needs no API key, no account and no indexer. +class HistoryConflict(SourceUnavailable): + """RPC event metadata contradicts this snapshot or earlier observations.""" + + +def _history_page(logs, token, start, end, head, head_hash): + page = {} + for raw in logs: + if not isinstance(raw, dict) or raw.get("removed", False) is not False: + raise SourceUnavailable("invalid or removed log") + if not is_hex_bytes(raw.get("address"), 20) or raw["address"].lower() != token.lower(): + raise SourceUnavailable("log belongs to another token") + number = _quantity(raw.get("blockNumber")) + index = _quantity(raw.get("logIndex")) + tx_index = _quantity(raw.get("transactionIndex")) + if not start <= number <= end or not is_hex_bytes(raw.get("blockHash"), 32): + raise SourceUnavailable("log is outside the requested range or has no block hash") + topics = raw.get("topics") + if not isinstance(topics, list) or len(topics) != 3 or any( + not is_hex_bytes(t, 32) for t in topics): + raise SourceUnavailable("invalid ERC20 Transfer topics") + if topics[0].lower() != TRANSFER_TOPIC: + raise SourceUnavailable("unexpected event signature") + if any(int(t[2:26], 16) for t in topics[1:]): + raise SourceUnavailable("nonzero address padding in Transfer topic") + normalized = dict(raw, topics=[t.lower() for t in topics]) + try: + decoded = decode_transfer(normalized) + except (ValueError, TypeError, KeyError) as exc: + raise SourceUnavailable(f"invalid Transfer: {exc}") from exc + decoded["transaction_hash"] = decoded["transaction_hash"].lower() + decoded["from"]["hash"] = decoded["from"]["hash"].lower() + decoded["to"]["hash"] = decoded["to"]["hash"].lower() + identity = (raw["blockHash"].lower(), index) + if number == head and raw["blockHash"].lower() != head_hash: + raise HistoryConflict("log at the fixed head has a different hash") + value = (number, tx_index, index, decoded) + if identity in page and page[identity] != value: + raise HistoryConflict("conflicting copies of one log") + page[identity] = value + return page + + +def derive_transfers(token: str, wallets, chain_id=8453): + """Continuous token-log coverage, never wallet-nonce sampling. + + Successful eth_getLogs responses are trusted RPC evidence, not receipt + proofs. Budgets or bad responses leave explicit incomplete coverage. """ - def nonce_at(address, block): - return int(_rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, - "method": "eth_getTransactionCount", - "params": [address, hex(block)]}), - "eth_getTransactionCount"), 16) - - head = int(_rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, - "method": "eth_blockNumber", "params": []}), - "eth_blockNumber"), 16) + try: + span = int(os.environ.get("BASE_HISTORY_BLOCK_SPAN", "10000")) + log_cap = int(os.environ.get("BASE_HISTORY_MAX_LOGS_PER_PAGE", "10000")) + total_log_cap = int(os.environ.get("BASE_HISTORY_MAX_LOGS", "100000")) + budget = RpcBudget( + int(os.environ.get("BASE_HISTORY_MAX_REQUESTS", "2048")), + float(os.environ.get("BASE_HISTORY_TIMEOUT", "300")), + int(os.environ.get("BASE_HISTORY_MAX_RESPONSE_BYTES", str(8 * 1024**2)))) + except (TypeError, ValueError, OverflowError) as exc: + raise SourceUnavailable("invalid history budget configuration") from exc + if span <= 0 or log_cap <= 0 or total_log_cap <= 0 or not is_hex_bytes(token, 20): + raise SourceUnavailable("invalid token or history page limits") + if type(chain_id) is not int or chain_id <= 0 or any( + not is_hex_bytes(w, 20) for w in wallets): + raise SourceUnavailable("invalid chain or project wallet") + for endpoint in (None, LOGS_RPC_URL): + if _quantity(_request(budget, "eth_chainId", [], endpoint)) != chain_id: + raise SourceUnavailable("RPC chain ID differs from ledger.chain_id") + header = _header(budget, "latest") + head = _quantity(header["number"]) if head < DEPLOYMENT_BLOCK: raise SourceUnavailable("chain head precedes the deployment block") + history = TransferHistory([], head, header["hash"].lower(), + DEPLOYMENT_BLOCK - 1, budget=budget) + log_header = _header(budget, hex(head), LOGS_RPC_URL) + if log_header["hash"].lower() != history.head_hash: + raise SourceUnavailable("RPC endpoints disagree about the fixed head") + cursor = DEPLOYMENT_BLOCK + seen, number_hash, hash_number = {}, {}, {} + # Reserve both endpoint head checks. Even after a page exhausts its + # request allowance, earlier pages can still be confirmed or rejected. + budget.reserve = 2 + try: + while cursor <= head: + end = min(head, cursor + span - 1) + logs = _request(budget, "eth_getLogs", [{ + "address": token, "topics": [TRANSFER_TOPIC], + "fromBlock": hex(cursor), "toBlock": hex(end), + }], LOGS_RPC_URL) + # A provider cap may mean an incomplete array even with HTTP200. + # Do not accept a capped page as complete or advance its cursor. + if len(logs) >= log_cap: + raise SourceUnavailable("log page reached the configured result cap") + page = _history_page(logs, token, cursor, end, head, history.head_hash) + staged_numbers, staged_hashes = dict(number_hash), dict(hash_number) + added = [] + for identity, value in page.items(): + block_hash, _ = identity + number = value[0] + if ((number in staged_numbers and staged_numbers[number] != block_hash) + or (block_hash in staged_hashes and staged_hashes[block_hash] != number)): + raise HistoryConflict("conflicting block number/hash in history") + if identity in seen and seen[identity] != value: + raise HistoryConflict("log identity changed between pages") + staged_numbers[number] = block_hash + staged_hashes[block_hash] = number + if identity not in seen: + added.append(value) + if len(seen) + len(added) > total_log_cap: + raise SourceUnavailable("history log-memory budget exhausted") + seen.update(page) + number_hash, hash_number = staged_numbers, staged_hashes + history.transfers.extend(value[3] for value in sorted(added, + key=lambda value: value[:3])) + history.covered_to = end + cursor = end + 1 + history.complete = True + except HistoryConflict as exc: + # A self-contradictory provider invalidates this run's observations; + # do not preserve a finding produced by the contradictory records. + history.error = str(exc) + return history + except SourceUnavailable as exc: + history.error = str(exc) + finally: + budget.reserve = 0 + try: + for endpoint in (None, LOGS_RPC_URL): + current = _header(budget, hex(head), endpoint) + if current["hash"].lower() != history.head_hash: + raise SourceUnavailable("fixed-head hash changed during history scan") + history.canonical = True + except SourceUnavailable as exc: + history.complete = False + history.error = str(exc) + return history - blocks = {DEPLOYMENT_BLOCK} # the mint itself - for address in sorted(wallets): - blocks.update(blocks_with_transactions( - address, DEPLOYMENT_BLOCK - 1, head, nonce_at)) - - transfers = [] - spenders = set() - for block in sorted(blocks): - # One unfiltered read per block gets Transfers and Approvals together; - # filtering by topic would cost a second request for no benefit. - logs = _rpc_result( - rpc_call({"jsonrpc": "2.0", "id": 1, "method": "eth_getLogs", - "params": [{"address": token, - "fromBlock": hex(block), - "toBlock": hex(block)}]}, - endpoint=LOGS_RPC_URL), - "eth_getLogs") - for log in logs: - if log.get("removed"): - continue # dropped by a reorg; not part of history - topics = log.get("topics") or [] - if not topics: - continue - if topics[0].lower() == TRANSFER_TOPIC.lower(): - transfers.append(decode_transfer(log)) - elif topics[0].lower() == APPROVAL_TOPIC.lower() and len(topics) == 3: - owner = "0x" + topics[1][-40:] - if owner.lower() in {w.lower() for w in wallets}: - spenders.add("0x" + topics[2][-40:]) - return transfers, spenders - - -def fetch_transfers(token: str, wallets): - """The complete transfer history, derived from the chain. - - There is one source and it is the chain. An indexer would answer faster, - but every one of them is somebody's service to withdraw — and a shortcut - that silently skips checks when it works is worse than no shortcut. If a - free, keyless indexer for Base reappears, it belongs here only if it - supplies the approvals too. - """ - transfers, spenders = derive_transfers(token, wallets) - LIVE_SPENDERS.extend(spenders) - return transfers + +def fetch_transfers(token: str, wallets, chain_id=8453): + return derive_transfers(token, wallets, chain_id) def _srcmnt(wei, signed=False): @@ -349,30 +490,33 @@ def _rpc_result(response, method): if not isinstance(result, list): raise SourceUnavailable(f"{method}: expected a list of logs") - elif method == "eth_blockNumber": + elif method in ("eth_blockNumber", "eth_chainId"): if ( not isinstance(result, str) or re.fullmatch(r"0x(?:0|[1-9a-fA-F][0-9a-fA-F]*)", result) is None ): raise SourceUnavailable(f"{method}: invalid block number") + elif method == "eth_getBlockByNumber": + if not isinstance(result, dict) or not is_hex_bytes(result.get("hash"), 32): + raise SourceUnavailable("invalid block header") + _quantity(result.get("number")) + elif method == "eth_getTransactionReceipt": + if result is not None and not isinstance(result, dict): + raise SourceUnavailable("invalid transaction receipt") else: raise SourceUnavailable(f"Unsupported method: {method}") return result -def verify_balances_total_supply(ledger, token, rpc): +def verify_balances_total_supply(ledger, token, rpc, block=None): wallets = ledger["project_wallets"] expected_state = calculate_expected_state(ledger) - response_block = rpc({ - "jsonrpc": "2.0", - "method": "eth_blockNumber", - "params": [], - "id": 1 - }) - block = _rpc_result(response_block, "eth_blockNumber") + if block is None: + block = _rpc_result(rpc({"jsonrpc": "2.0", "method": "eth_blockNumber", + "params": [], "id": 1}), "eth_blockNumber") for name, address in wallets.items(): address = address.lower() @@ -451,7 +595,10 @@ def main() -> int: checked = 0 try: - transfers = fetch_transfers(token, wallets) + history = fetch_transfers(token, wallets, ledger["chain_id"]) + if not isinstance(history, TransferHistory): + raise SourceUnavailable("history source did not provide coverage metadata") + transfers = history.transfers except SourceUnavailable as e: print("COULD NOT VERIFY: no transfer history could be obtained.") print(f" tried: {e}") @@ -459,6 +606,11 @@ def main() -> int: print(" The next successful run re-checks the full history.") return 2 + if not history.canonical: + print("COULD NOT VERIFY: fixed-head history was not confirmed canonical.") + print(f" {history.error}") + return 2 + for t in transfers: frm = t["from"]["hash"].lower() to = t["to"]["hash"].lower() @@ -501,64 +653,98 @@ def main() -> int: print("\nLEDGER entries not found on-chain:") print("\n".join(missing_onchain)) - # Everything above checks movements we signed. A third party spending an - # allowance moves tokens without touching our nonce and without appearing - # in a block we looked at — the exact complement. A balance that the - # ledger cannot explain is what that looks like. - balance_violation = False - try: - verify_balances_total_supply(ledger, token, rpc_call) - except AccountingViolation as e: - ok = False - balance_violation = True - print("\nBALANCE UNEXPLAINED (a movement we did not sign):") - print(f" {e}") - except SourceUnavailable as e: - # A violation already found is a finding; an outage must never - # downgrade it to "could not verify". - if not ok: - print(f"\nnote: balances unchecked ({e})") - else: - print("COULD NOT VERIFY: transfers matched, but balances could " - f"not be read to rule out a movement we did not sign ({e}).") - return 2 - - # An outstanding allowance lets tokens leave without our wallet signing, - # in a block this derivation never reads. Claiming completeness while one - # is live would be claiming more than was checked. - outstanding = [] - for spender in sorted(set(LIVE_SPENDERS)): - for name, wallet in ledger["project_wallets"].items(): + print(f"history coverage: {DEPLOYMENT_BLOCK}..{history.covered_to} " + f"of fixed head {history.head} ({history.head_hash})") + if not history.complete: + print(f"history incomplete: {history.error}") + if unaccounted: + print("\nRESULT: FAIL — confirmed unaccounted movement in covered history.") + return 1 + print("COULD NOT VERIFY: unobserved entries are not findings in partial history.") + return 2 + + # An absent entry can refer to a pending or post-snapshot transaction. + # Only a canonical receipt inside this covered interval makes absence a + # finding; null, future or unavailable receipts leave it inconclusive. + unresolved = False + missing_finding = False + def state_rpc(payload): + return history.budget.call(rpc_call, payload) if history.budget else rpc_call(payload) + + if history.budget: + history.budget.reserve = 2 # final fixed-head checks cannot be spent on state/retries + if missing_onchain: + for tx in {key[0] for key in entries if entries[key] > observed[key]}: try: - raw = _rpc_result(rpc_call({ - "jsonrpc": "2.0", "id": 1, "method": "eth_call", - "params": [{"to": token, - "data": ALLOWANCE_SELECTOR - + wallet[2:].lower().zfill(64) - + spender[2:].lower().zfill(64)}, - "latest"]}), "allowance") - except SourceUnavailable as e: - # A missing sampled entry is not an observed violation, but - # an actual transfer or balance finding survives an outage. - if unaccounted or balance_violation: - print(f"\nnote: allowance for {spender} unchecked ({e}).") - return 1 - print(f"\nCOULD NOT VERIFY: allowance for {spender} unreadable ({e}).") - return 2 - if int(raw, 16): - outstanding.append((name, spender, int(raw, 16))) - - if outstanding: - ok = False - print("\nOUTSTANDING ALLOWANCES (tokens can leave without us signing):") - for name, spender, amount in outstanding: - shown = "unlimited" if amount > 2**255 else f"{amount // DECIMALS:,}" - print(f" {name} -> {spender} : {shown} SRCMNT") - print(" While these stand, transfers made by the spender do not appear") - print(" in any block this check reads. Revoke them or the history is") - print(" not provably complete.") - - print("\nRESULT:", "OK — every project-wallet movement is accounted for." if ok else "FAIL") + receipt = _rpc_result(state_rpc({"jsonrpc": "2.0", "id": 1, + "method": "eth_getTransactionReceipt", "params": [tx]}), + "eth_getTransactionReceipt") + if receipt is None: + unresolved = True + continue + number = _quantity(receipt.get("blockNumber")) + if (not is_hex_bytes(receipt.get("transactionHash"), 32) or + receipt["transactionHash"].lower() != tx or + not is_hex_bytes(receipt.get("blockHash"), 32)): + raise SourceUnavailable("receipt identity is invalid") + if not DEPLOYMENT_BLOCK <= number <= history.head: + unresolved = True + continue + header = _rpc_result(state_rpc({"jsonrpc": "2.0", "id": 1, + "method": "eth_getBlockByNumber", "params": [hex(number), False]}), + "eth_getBlockByNumber") + if (_quantity(header["number"]) != number or + header["hash"].lower() != receipt["blockHash"].lower()): + unresolved = True + else: + missing_finding = True + except SourceUnavailable as exc: + print(f"note: missing-entry receipt unchecked ({exc})") + unresolved = True + ok = not (unaccounted or missing_finding) + + # A pending/future ledger item does not belong to the H-time state. + # Never compare its nonzero amount against H and call that a violation. + state_unavailable = False + if not unresolved: + try: + verify_balances_total_supply(ledger, token, state_rpc, history.block_ref) + except AccountingViolation as exc: + ok = False + print("\nBALANCE UNEXPLAINED:") + print(f" {exc}") + except SourceUnavailable as exc: + state_unavailable = True + print(f"note: snapshot balances unchecked ({exc})") + + # Receipts and state must not introduce findings from a later fork. + # Confirm H again after those reads, including after state errors. + if history.budget: + history.budget.reserve = 0 + try: + for endpoint in (None, LOGS_RPC_URL): + payload = {"jsonrpc": "2.0", "id": 1, + "method": "eth_getBlockByNumber", "params": [hex(history.head), False]} + response = (history.budget.call(rpc_call, payload, endpoint) + if history.budget else rpc_call(payload, endpoint=endpoint)) + current = _rpc_result(response, "eth_getBlockByNumber") + if (_quantity(current["number"]) != history.head or + current["hash"].lower() != history.head_hash): + raise SourceUnavailable("fixed-head identity changed after receipt/state reads") + except SourceUnavailable as exc: + print(f"COULD NOT VERIFY: final snapshot identity unavailable ({exc}).") + return 2 + + if unresolved and not unaccounted and not missing_finding: + print("COULD NOT VERIFY: ledger entries are outside or unresolved at the snapshot.") + return 2 + if state_unavailable and ok: + print("COULD NOT VERIFY: no finding established and snapshot state unreadable.") + return 2 + # Complete Transfer coverage includes transferFrom irrespective of current + # allowance. A live allowance is future authority, not past missing history. + print("\nRESULT:", "OK — ledger matches trusted RPC history at the fixed head." + if ok else "FAIL") return 0 if ok else 1