diff --git a/src/lean_spec/spec/forks/lstar/block_production.py b/src/lean_spec/spec/forks/lstar/block_production.py index bdc9bac1c..3da437bdb 100644 --- a/src/lean_spec/spec/forks/lstar/block_production.py +++ b/src/lean_spec/spec/forks/lstar/block_production.py @@ -2,6 +2,8 @@ from collections import defaultdict from collections.abc import Set as AbstractSet +from dataclasses import dataclass +from enum import IntEnum from lean_spec.spec.crypto.merkleization import hash_tree_root from lean_spec.spec.crypto.xmss.containers import PublicKey @@ -14,13 +16,54 @@ AggregatedAttestation, AttestationData, Block, - Checkpoint, + JustifiedSlots, SingleMessageAggregate, Slot, State, ValidatorIndex, ) -from lean_spec.spec.ssz import ZERO_HASH, Bytes32 +from lean_spec.spec.ssz import ZERO_HASH, Boolean, Bytes32 + + +class _Tier(IntEnum): + """ + Selection tier for a candidate attestation data entry. + + Declared in priority order: a lower value wins. + """ + + FINALIZE = 1 + """Applying the entry crosses two-thirds on target and finalizes the source.""" + JUSTIFY = 2 + """Applying the entry crosses two-thirds on target but does not finalize.""" + BUILD = 3 + """Adds marginal new voters toward target's two-thirds supermajority.""" + + +@dataclass(frozen=True) +class _EntryScore: + """ + Tiered score for a candidate attestation data entry during block building. + + Lower tier wins. + Within a tier, more new voters wins, then a smaller target slot, then a + smaller attestation slot, then the entry's data root for determinism. + """ + + tier: _Tier + new_voter_count: int + target_slot: Slot + attestation_slot: Slot + + def ordering_key(self, data_root: Bytes32) -> tuple[int, int, int, int, bytes]: + """Sort key where the smallest tuple is the best candidate.""" + return ( + int(self.tier), + -self.new_voter_count, + int(self.target_slot), + int(self.attestation_slot), + bytes(data_root), + ) class BlockProductionMixin(LstarSpecBase): @@ -53,16 +96,13 @@ def build_block( Each round repeats these steps: - 1. Pick the eligible proofs covering the most uncounted validators. - 2. Apply the state transition. - 3. Re-anchor on any newly justified checkpoint. - - The rounds stop once a pass adds nothing. - - # Why it terminates + 1. Score every remaining candidate against the projected post-state. + 2. Pick the best one: finalize beats justify beats build. + 3. Project justification and finalization forward, unlocking dependents. - Justification and finalization only move forward, and the chosen set only grows. - Both are bounded, so the rounds must end. + The rounds stop at the data-entry cap or once nothing scores. + Projection replaces trial state transitions, so the real transition + runs only once at the end to seal the state root. Args: state: Pre-state the block builds on. @@ -83,24 +123,17 @@ def build_block( advanced_state = self.process_slots(state, slot) if aggregated_payloads: - # Anchor on the checkpoint this chain treats as justified. - # - # On genesis the parent is justified at slot 0 by header processing. - # Anchor there so eligible sources match. - current_justified_checkpoint = ( - Checkpoint(slot=Slot(0), root=parent_root) - if state.latest_block_header.slot == Slot(0) - else state.latest_justified - ) - - # Track which slots are already justified. + # Tiered greedy selection. # - # Extend the window so every slot the loop may query is covered. - # It spans the finalized boundary up to the slot before this block. - current_finalized_slot = state.latest_finalized.slot - current_justified_slots = state.justified_slots.extend_to_slot( - current_finalized_slot, slot - Slot(1) - ) + # Each round scores remaining candidates against a projected post-state + # and picks the best: finalize beats justify beats build. + # Justification and finalization are projected incrementally so dependent + # attestations become eligible on the next round without re-running the + # state transition. + # Selection stops at the data-entry cap or when no remaining candidate scores. + selected_attestations_with_proofs: list[ + tuple[AggregatedAttestation, SingleMessageAggregate] + ] = [] # Assemble the chain as it will look once this block is applied. # @@ -108,129 +141,217 @@ def build_block( # 2. The parent root at its own slot. # 3. A zero hash for each slot skipped before this block. # - # Source and target roots are validated against this view. - num_empty_slots = int(slot - state.latest_block_header.slot - Slot(1)) + # Precondition: the new slot must lie strictly after the parent slot. + # Without the guard, unsigned subtraction underflows and the empty-slot + # padding allocates an astronomically large list. + parent_slot = state.latest_block_header.slot + assert slot > parent_slot, ( + f"Cannot build block at slot {slot} <= parent slot {parent_slot}" + ) + num_empty_slots = int(slot - parent_slot - Slot(1)) extended_historical_block_hashes: list[Bytes32] = ( list(state.historical_block_hashes) + [parent_root] + [ZERO_HASH] * num_empty_slots ) + validator_count = len(state.validators) - processed_attestation_data: set[AttestationData] = set() - - # Order candidates by target slot, once. - candidates_in_target_slot_order = sorted( - aggregated_payloads.items(), key=lambda item: item[0].target.slot - ) + # Projected post-state, updated incrementally as entries are selected. + finalized_slot = state.latest_finalized.slot + justified_slots = state.justified_slots.extend_to_slot(finalized_slot, slot - Slot(1)) - # Fixed-point selection. + # Seed the running voter map from the on-chain justification bitlist. # - # - Each pass scans every candidate once, in target-slot order. - # - Accepting an entry may advance justification and unlock more. - # - Re-scan until a pass finds nothing new. - while True: - found_new_entries = False - - for attestation_data, proofs in candidates_in_target_slot_order: - if attestation_data in processed_attestation_data: - continue - - # Stop once the block holds the maximum distinct data entries. - # This cap is a proposer-side budget, not a consensus rule. - if len(processed_attestation_data) >= int(MAX_ATTESTATIONS_DATA): - break + # The state stores one bit per tracked-root and validator pair. + # The bit at index (root_index * N + validator_index) means that + # validator voted for that tracked root, where N is the validator count. + # Seeding from these bits lets scoring count on-chain voters toward the + # two-thirds threshold. + votes_by_target_root: dict[Bytes32, set[ValidatorIndex]] = {} + for root_index, on_chain_target_root in enumerate(state.justifications_roots): + votes_by_target_root[on_chain_target_root] = { + ValidatorIndex(validator_index) + for validator_index in range(validator_count) + if state.justifications_validators[ + root_index * validator_count + validator_index + ] + } + processed_attestation_data: set[AttestationData] = set() - # Skip votes whose head block the proposer has not seen. - if attestation_data.head.root not in known_block_roots: + for _ in range(int(MAX_ATTESTATIONS_DATA)): + # Scan every remaining candidate and keep the best ordering key. + # + # Skips entries already processed, those failing the projected-chain + # filters, and those with zero new voters. + # A smaller ordering key wins. + best_candidate: tuple[AttestationData, _EntryScore, set[ValidatorIndex]] | None = ( + None + ) + best_candidate_key: tuple[int, int, int, int, bytes] | None = None + for candidate_data, proofs in aggregated_payloads.items(): + if candidate_data in processed_attestation_data: continue - # Reject votes that do not match this chain. + # Validate the candidate against the projected chain view. + # + # Mirrors the vote-validity rules: head must be known, source must be + # justified, source and target must match the candidate-block chain view, + # target must be after source, target must not already be justified, and + # target must be justifiable relative to the projected finalized slot. # - # This also rejects any checkpoint past the chain view. - # That keeps the bounded lookups below in range. - if not attestation_data.lies_on_chain(extended_historical_block_hashes): + # Chain-match runs before the justified-slot queries: it rejects + # checkpoints whose slot is past the chain view, which keeps the bounded + # justification queries from raising IndexError. + if candidate_data.head.root not in known_block_roots: continue - - # A vote may only build from an already-justified source. - if not current_justified_slots.is_slot_justified( - current_finalized_slot, attestation_data.source.slot + if not candidate_data.lies_on_chain(extended_historical_block_hashes): + continue + if not justified_slots.is_slot_justified( + finalized_slot, candidate_data.source.slot ): continue - # Genesis self-votes have source and target both at slot 0. - # - # - The state transition drops them: they justify nothing. - # - They still carry head weight for fork choice. - # - Including them propagates them to peers. - # - Slot 0 counts as justified, so the next check would drop them. - # - This flag lets them through. - source_at_genesis = attestation_data.source.slot == Slot(0) - target_at_genesis = attestation_data.target.slot == Slot(0) - is_genesis_self_vote = source_at_genesis and target_at_genesis - - # Skip votes whose target slot is already justified. - # - # - A justified target gains nothing from more votes. - # - Genesis self-votes are exempt, kept for their head weight. - if not is_genesis_self_vote and current_justified_slots.is_slot_justified( - current_finalized_slot, attestation_data.target.slot - ): + # A genesis self-vote anchors both source and target at slot 0. + # It cannot justify or finalize, but it carries fork-choice signal, + # so selection treats it specially. + is_genesis_self_vote = candidate_data.source.slot == Slot( + 0 + ) and candidate_data.target.slot == Slot(0) + + # Genesis self-votes are exempt from the target-after-source and + # target-already-justified checks. + # The state transition drops them, but they carry fork-choice signal. + if not is_genesis_self_vote: + if candidate_data.target.slot <= candidate_data.source.slot: + continue + if justified_slots.is_slot_justified( + finalized_slot, candidate_data.target.slot + ): + continue + if not candidate_data.target.slot.is_justifiable_after(finalized_slot): + continue + + # New voters: participants across all proofs not already recorded + # for the target. + prior_voters = votes_by_target_root.get(candidate_data.target.root, set()) + new_voters: set[ValidatorIndex] = set() + for proof in proofs: + for validator_index in proof.participants.to_validator_indices(): + if validator_index not in prior_voters: + new_voters.add(validator_index) + + # An entry adding no validators cannot improve the target, so skip it. + if not new_voters: continue - processed_attestation_data.add(attestation_data) - found_new_entries = True + # Threshold: total voters (prior plus new) crossing two-thirds. + total_voters = len(prior_voters) + len(new_voters) + crosses_two_thirds = 3 * total_voters >= 2 * validator_count - # Choose proofs covering the most validators. - # Emit one attestation per chosen proof. - selected_proofs, _ = select_proofs_for_coverage(proofs) - aggregated_signatures.extend(selected_proofs) - for proof in selected_proofs: - aggregated_attestations.append( - self.aggregated_attestation_class( - aggregation_bits=proof.participants, - data=attestation_data, + # 3SF-mini finalization requires no slot strictly between source and + # target to still be justifiable. + # Source and target must be consecutive justified checkpoints in the + # projected post-state. + # + # The source must lie strictly past the projected finalized boundary. + # A source at or behind the boundary is already final. + # It may still justify a newer target, but it must not re-finalize. + # This mirrors the state transition, which advances finalization only + # when the source slot is strictly greater than the finalized slot. + # Scanning from one past the source also keeps every queried slot + # strictly above the boundary, where justifiability is defined. + finalizes_source = ( + crosses_two_thirds + and candidate_data.source.slot > finalized_slot + and all( + not Slot(intermediate_slot).is_justifiable_after(finalized_slot) + for intermediate_slot in range( + int(candidate_data.source.slot) + 1, + int(candidate_data.target.slot), ) ) + ) + + # A genesis self-vote cannot justify or finalize and is always BUILD tier. + if is_genesis_self_vote or not crosses_two_thirds: + tier = _Tier.BUILD + elif finalizes_source: + tier = _Tier.FINALIZE + else: + tier = _Tier.JUSTIFY + + candidate_score = _EntryScore( + tier=tier, + new_voter_count=len(new_voters), + target_slot=candidate_data.target.slot, + attestation_slot=candidate_data.slot, + ) + candidate_key = candidate_score.ordering_key(hash_tree_root(candidate_data)) + if best_candidate_key is None or candidate_key < best_candidate_key: + best_candidate = (candidate_data, candidate_score, new_voters) + best_candidate_key = candidate_key - if not found_new_entries: + if best_candidate is None: break + attestation_data, entry_score, selected_new_voters = best_candidate + processed_attestation_data.add(attestation_data) - # Apply the state transition to a trial block. - # Its post-state reveals whether this pass advanced justification. - candidate_block = self.block_class( - slot=slot, - proposer_index=proposer_index, - parent_root=parent_root, - state_root=Bytes32.zero(), - body=self.block_body_class( - attestations=self.aggregated_attestations_class( - data=aggregated_attestations - ) - ), + # Pack proofs that maximize new validator coverage for this entry. + selected_proofs, _ = select_proofs_for_coverage( + aggregated_payloads[attestation_data] ) - post_state = self.process_block(advanced_state, candidate_block) + for proof in selected_proofs: + selected_attestations_with_proofs.append( + ( + self.aggregated_attestation_class( + aggregation_bits=proof.participants, + data=attestation_data, + ), + proof, + ) + ) - # Repeat only if justification or finalization moved. - # - # - Both advance monotonically, so the loop is bounded. - # - A finalization step slides the justified window forward. - # - That can make previously out-of-range targets eligible. - if ( - post_state.latest_justified != current_justified_checkpoint - or post_state.latest_finalized.slot != current_finalized_slot - ): - current_justified_checkpoint = post_state.latest_justified - current_justified_slots = post_state.justified_slots - current_finalized_slot = post_state.latest_finalized.slot - - # Re-anchoring needs no other rebuilds. - # The justified window still covers every slot the loop queries. - # The chain view is fixed once written, never recomputed. - continue - - break + target_root = attestation_data.target.root + + # Project justification and finalization. Finalize implies justify. + if entry_score.tier <= _Tier.JUSTIFY: + justified_slots = justified_slots.extend_to_slot( + finalized_slot, attestation_data.target.slot + ) + + # The justifiable filter and the extension above guarantee an + # in-range index, so mark the target justified directly. + target_justified_index = attestation_data.target.slot.justified_index_after( + finalized_slot + ) + assert target_justified_index is not None + updated_justified_bits = list(justified_slots.data) + updated_justified_bits[target_justified_index] = Boolean(True) + justified_slots = JustifiedSlots(data=updated_justified_bits) + + # A justified target can no longer be a candidate target, so its + # voter bucket is irrelevant for further scoring. + votes_by_target_root.pop(target_root, None) + else: + # BUILD tier: the target stays a candidate, so record its new + # voters to push it toward the threshold on a later round. + votes_by_target_root.setdefault(target_root, set()).update(selected_new_voters) + if entry_score.tier == _Tier.FINALIZE: + # The finalize tier requires a source strictly past the boundary, + # so the window always advances by at least one slot. + # Drop the leading bits that fell behind the new finalized boundary. + new_finalized_slot = attestation_data.source.slot + finalized_slot_advance = int(new_finalized_slot) - int(finalized_slot) + justified_slots = JustifiedSlots( + data=justified_slots.data[finalized_slot_advance:] + ) + finalized_slot = new_finalized_slot + + for attestation, proof in selected_attestations_with_proofs: + aggregated_attestations.append(attestation) + aggregated_signatures.append(proof) # Collapse each attestation data down to a single proof. # - # - The coverage picker may emit several proofs for one data in a pass. + # - The coverage picker may emit several proofs for one data entry. # - A block must carry one attestation per data, over the union of voters. # Group every proof under the data it attests to. @@ -291,10 +412,7 @@ def build_block( ), ) - # Recompute the post-state to obtain the state root. - # - # Merging proofs keeps the same voters, so the post-state is unchanged. - # Only the body's shape differs, so just the root is needed. + # Compute the post-state to obtain the state root. post_state = self.process_block(advanced_state, final_block) final_block = final_block.model_copy(update={"state_root": hash_tree_root(post_state)}) diff --git a/src/lean_spec/spec/forks/lstar/validator_duties.py b/src/lean_spec/spec/forks/lstar/validator_duties.py index ba080032f..7f46a5c2e 100644 --- a/src/lean_spec/spec/forks/lstar/validator_duties.py +++ b/src/lean_spec/spec/forks/lstar/validator_duties.py @@ -117,8 +117,8 @@ def produce_block_with_signatures( 3. Build the block with maximal valid attestations 4. Store the block and update checkpoints - The block builder uses a fixed-point algorithm to collect attestations. - Each iteration may update the justified checkpoint. + The block builder uses a tiered greedy scorer to collect attestations. + Each round projects justification forward, unlocking dependent entries. Returns the per-attestation single-message aggregate proofs unmerged. The validator service signs the block root with the proposal key, wraps that into @@ -154,9 +154,9 @@ def produce_block_with_signatures( # Build the block. # - # The builder iteratively collects valid attestations from aggregated - # payloads matching the justified checkpoint. Each iteration may advance - # justification, unlocking more attestation data entries. + # The builder scores valid attestations from aggregated payloads against + # a projected post-state. Each round projects justification forward, + # unlocking more attestation data entries. final_block, final_post_state, _, signatures = self.build_block( head_state, slot=slot, @@ -169,8 +169,8 @@ def produce_block_with_signatures( # Invariant: the produced block must close any justified divergence. # # The store may have advanced its justified checkpoint from attestations - # on a minority fork that the head state never processed. The fixed-point - # loop above must incorporate those attestations from the pool, advancing + # on a minority fork that the head state never processed. The selection + # above must incorporate those attestations from the pool, advancing # the block's justified checkpoint to at least match the store. # # Without this, other nodes processing the block would never see the @@ -180,7 +180,7 @@ def produce_block_with_signatures( store_justified = store.latest_justified.slot assert block_justified >= store_justified, ( f"Produced block justified={block_justified} < store justified=" - f"{store_justified}. Fixed-point attestation loop did not converge." + f"{store_justified}. Attestation selection did not close the divergence." ) # Compute block hash for storage. diff --git a/tests/consensus/lstar/fork_choice/test_attestation_source_divergence.py b/tests/consensus/lstar/fork_choice/test_attestation_source_divergence.py index 689788ada..8c310cd45 100644 --- a/tests/consensus/lstar/fork_choice/test_attestation_source_divergence.py +++ b/tests/consensus/lstar/fork_choice/test_attestation_source_divergence.py @@ -42,6 +42,9 @@ def test_justified_divergence_self_heals_in_next_block( Then ---- - block_5 pulls the slot-1 votes from the pool and includes them. + - V0's slot-2 vote is already recorded on the head chain. + - that vote adds no new voters, so the builder omits it. + - the block body holds 1 aggregated vote. - the head chain justifies slot 1, matching the node. - finalized stays at slot 0. """ @@ -103,16 +106,12 @@ def test_justified_divergence_self_heals_in_next_block( head_root_label="block_5", latest_justified_slot=Slot(1), latest_justified_root_label="common", - block_attestation_count=2, + block_attestation_count=1, block_attestations=[ AggregatedAttestationCheck( participants={1, 2, 3}, target_slot=Slot(1), ), - AggregatedAttestationCheck( - participants={0}, - target_slot=Slot(2), - ), ], ), ), diff --git a/tests/consensus/lstar/fork_choice/test_block_production.py b/tests/consensus/lstar/fork_choice/test_block_production.py index fc72e4fa1..5210c96ca 100644 --- a/tests/consensus/lstar/fork_choice/test_block_production.py +++ b/tests/consensus/lstar/fork_choice/test_block_production.py @@ -188,12 +188,14 @@ def test_produce_block_enforces_max_attestations_data_limit( Given ----- - - 3 attesting validators. - the chain: - genesis -> block_1(1) -> ... -> one block past the limit - - one vote per target block arrives by gossip. + genesis -> block_1(1) -> ... -> the highest justifiable target slot + - one vote from V0 per justifiable target slot arrives by gossip. - each vote names a different target. - this yields one more distinct attestation data entry than the limit allows. + - only justifiable targets are used, so every entry is a real candidate. + - a single voter never reaches the supermajority (2/3). + - no entry justifies its target, so the cap alone bounds the count. When ---- @@ -201,8 +203,8 @@ def test_produce_block_enforces_max_attestations_data_limit( Then ---- - - the builder sorts entries by target slot and stops at the limit. - - the entries with the highest target slots are dropped. + - same-tier entries are ordered by target slot. + - the entry with the highest target slot is dropped. - the produced block holds exactly the maximum number of votes. Timing @@ -212,32 +214,43 @@ def test_produce_block_enforces_max_attestations_data_limit( - a tick to the next slot start moves the votes into the known pool. """ limit = int(MAX_ATTESTATIONS_DATA) - num_target_blocks = limit + 1 - block_production_slot = num_target_blocks + 1 - validators = [ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)] - aggregate_interval = num_target_blocks * int(INTERVALS_PER_SLOT) + 2 + # The first limit + 1 slots that are justifiable after genesis. + # One more than the cap, so exactly one candidate must be dropped. + target_slots: list[int] = [] + candidate_slot = 1 + while len(target_slots) < limit + 1: + if Slot(candidate_slot).is_justifiable_after(Slot(0)): + target_slots.append(candidate_slot) + candidate_slot += 1 + + # Build a contiguous chain up to the highest target slot, then produce one + # slot later. Every target block must exist on-chain to be a valid target. + chain_length = target_slots[-1] + block_production_slot = chain_length + 1 + + aggregate_interval = chain_length * int(INTERVALS_PER_SLOT) + 2 aggregate_time = math.ceil(aggregate_interval * int(MILLISECONDS_PER_INTERVAL) / 1000) next_slot_time = block_production_slot * int(SECONDS_PER_SLOT) chain_steps: list[BlockStep] = [ BlockStep( block=BlockSpec(slot=Slot(n), label=f"block_{n}"), - checks=(StoreChecks(head_slot=Slot(n)) if n == 1 or n == num_target_blocks else None), + checks=(StoreChecks(head_slot=Slot(n)) if n == 1 or n == chain_length else None), ) - for n in range(1, num_target_blocks + 1) + for n in range(1, chain_length + 1) ] attestation_steps: list[GossipAggregatedAttestationStep] = [ GossipAggregatedAttestationStep( attestation=AggregatedAttestationSpec( - validator_indices=validators, - slot=Slot(num_target_blocks), + validator_indices=[ValidatorIndex(0)], + slot=Slot(chain_length), target_slot=Slot(n), target_root_label=f"block_{n}", ), ) - for n in range(1, num_target_blocks + 1) + for n in target_slots ] fork_choice_test(