From df244594ab91e7f66ef3480aaff38e33b0021b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:24:17 -0300 Subject: [PATCH 1/2] perf(beacon): fuse electra epoch steps 2-9 into one registry pass Electra and fulu walk the tree-backed registry once per epoch step: inactivity updates, rewards, registry updates, slashings, pending deposits and effective-balance updates, several of them with a validator(i) descent per active validator. At mainnet scale that is most of an epoch-start block. stf::epoch::single_pass visits each validator once, in index order, on a local copy of it and of its balance, and writes the changes back afterwards. What depends on more than one validator stays outside the loop: the exit-churn cursor is advanced locally in index order, the pending-deposit queue is planned before the loop (top-ups of existing validators inside it, deposits that create validators after it, one registry walk for all pubkey lookups), and pending consolidations run after it with the effective-balance update of every validator they name deferred until they are done. The total active balance is the summary's, which equals get_total_active_balance through step 8. The genesis epoch and states whose registry-sized lists differ in length fall back to the step-by-step path. The step functions stay public and callable in isolation for the fixtures; the per-step rules the loop shares with them (registry action, slashing penalty, effective-balance hysteresis, inactivity score, exit-cursor advance) were factored out so both paths run the same code. In debug builds a registry of up to 4096 validators is also run through the specification-shaped steps on a clone and the two full states must hash equal. A randomized test compares the fused and unfused paths on crafted electra and fulu states. --- .../src/beacon/helpers/electra.rs | 163 +++-- .../src/beacon/stf/epoch/altair.rs | 59 +- .../src/beacon/stf/epoch/electra.rs | 233 ++++-- .../src/beacon/stf/epoch/mod.rs | 3 + .../src/beacon/stf/epoch/single_pass.rs | 678 ++++++++++++++++++ .../stf/epoch/single_pass_equivalence.rs | 553 ++++++++++++++ docs/beacon_stf.md | 13 + 7 files changed, 1562 insertions(+), 140 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs create mode 100644 crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index b2afcf7c..f6399653 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -352,11 +352,20 @@ pub fn get_max_effective_balance(validator: &Validator) -> Gwei { /// `EFFECTIVE_BALANCE_INCREMENT` so the budget always divides evenly into the /// unit every balance change is already rounded to. pub fn get_balance_churn_limit(state: &BeaconState, config: &Config) -> Result { - let total_active_balance = get_total_active_balance(state)?; + Ok(balance_churn_limit_for( + get_total_active_balance(state)?, + config, + )) +} + +/// [`get_balance_churn_limit`] for a caller that already holds the total +/// active balance, so the epoch's single pass does not rescan the registry +/// for it. +pub(crate) fn balance_churn_limit_for(total_active_balance: Gwei, config: &Config) -> Gwei { let churn = config .min_per_epoch_churn_limit_electra .max(total_active_balance / config.churn_limit_quotient); - Ok(churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT) + churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT } /// The portion of [`get_balance_churn_limit`] set aside for activations and @@ -367,9 +376,18 @@ pub fn get_balance_churn_limit(state: &BeaconState, config: &Config) -> Result Result { - Ok(config + Ok(activation_exit_churn_limit_for( + get_total_active_balance(state)?, + config, + )) +} + +/// [`get_activation_exit_churn_limit`] for a caller that already holds the +/// total active balance. +pub(crate) fn activation_exit_churn_limit_for(total_active_balance: Gwei, config: &Config) -> Gwei { + config .max_per_epoch_activation_exit_churn_limit - .min(get_balance_churn_limit(state, config)?)) + .min(balance_churn_limit_for(total_active_balance, config)) } /// The portion of [`get_balance_churn_limit`] left over for consolidations @@ -583,57 +601,106 @@ pub fn compute_exit_epoch_and_update_churn( let mut fields = electra_state(state, "compute_exit_epoch_and_update_churn")?; - let mut earliest_exit_epoch = fields - .earliest_exit_epoch() - .max(compute_activation_exit_epoch(current_epoch)); - - // A later epoch than the cursor currently sits on: that epoch has not - // spent any of its churn yet, so its budget starts full. Otherwise the - // cursor has not moved, and whatever it left unspent carries over. - let mut exit_balance_to_consume = if fields.earliest_exit_epoch() < earliest_exit_epoch { - per_epoch_churn - } else { - fields.exit_balance_to_consume() + let mut cursor = ExitChurnCursor { + earliest_exit_epoch: fields.earliest_exit_epoch(), + exit_balance_to_consume: fields.exit_balance_to_consume(), }; + let exit_epoch = cursor.advance(exit_balance, per_epoch_churn, current_epoch)?; + *fields.exit_balance_to_consume_mut() = cursor.exit_balance_to_consume; + *fields.earliest_exit_epoch_mut() = cursor.earliest_exit_epoch; - if exit_balance > exit_balance_to_consume { - let balance_to_process = exit_balance - exit_balance_to_consume; - // Ceiling division: how many additional epochs' worth of churn this - // exit needs beyond what the current epoch has left. - let additional_epochs = balance_to_process - .checked_sub(1) - .and_then(|value| value.checked_div(per_epoch_churn)) - .and_then(|value| value.checked_add(1)) - .ok_or(Error::ArithmeticOverflow( - "(exit_balance - exit_balance_to_consume - 1) / per_epoch_churn + 1", - ))?; - let additional_churn = - additional_epochs - .checked_mul(per_epoch_churn) + Ok(exit_epoch) +} + +/// Electra's exit-queue cursor, the pair of state fields +/// [`compute_exit_epoch_and_update_churn`] advances. +/// +/// Held by value so the epoch's single pass can advance it locally, in +/// registry order, without a state borrow per ejection, and write it back +/// once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ExitChurnCursor { + pub(crate) earliest_exit_epoch: Epoch, + pub(crate) exit_balance_to_consume: Gwei, +} + +impl ExitChurnCursor { + /// The cursor as `state` holds it (an electra or fulu state). + pub(crate) fn read(state: &mut BeaconState) -> Result { + let fields = electra_state(state, "ExitChurnCursor::read")?; + Ok(Self { + earliest_exit_epoch: fields.earliest_exit_epoch(), + exit_balance_to_consume: fields.exit_balance_to_consume(), + }) + } + + /// Stores the cursor back into `state`. + pub(crate) fn write(self, state: &mut BeaconState) -> Result<()> { + let mut fields = electra_state(state, "ExitChurnCursor::write")?; + *fields.earliest_exit_epoch_mut() = self.earliest_exit_epoch; + *fields.exit_balance_to_consume_mut() = self.exit_balance_to_consume; + Ok(()) + } + + /// The body of [`compute_exit_epoch_and_update_churn`]: advances the + /// cursor for an exit of `exit_balance` and returns the epoch it takes + /// effect at. On error the cursor is left untouched. + pub(crate) fn advance( + &mut self, + exit_balance: Gwei, + per_epoch_churn: Gwei, + current_epoch: Epoch, + ) -> Result { + let mut earliest_exit_epoch = self + .earliest_exit_epoch + .max(compute_activation_exit_epoch(current_epoch)); + + // A later epoch than the cursor currently sits on: that epoch has not + // spent any of its churn yet, so its budget starts full. Otherwise the + // cursor has not moved, and whatever it left unspent carries over. + let mut exit_balance_to_consume = if self.earliest_exit_epoch < earliest_exit_epoch { + per_epoch_churn + } else { + self.exit_balance_to_consume + }; + + if exit_balance > exit_balance_to_consume { + let balance_to_process = exit_balance - exit_balance_to_consume; + // Ceiling division: how many additional epochs' worth of churn this + // exit needs beyond what the current epoch has left. + let additional_epochs = balance_to_process + .checked_sub(1) + .and_then(|value| value.checked_div(per_epoch_churn)) + .and_then(|value| value.checked_add(1)) .ok_or(Error::ArithmeticOverflow( - "additional_epochs * per_epoch_churn", + "(exit_balance - exit_balance_to_consume - 1) / per_epoch_churn + 1", ))?; - earliest_exit_epoch = - earliest_exit_epoch - .checked_add(additional_epochs) + let additional_churn = + additional_epochs + .checked_mul(per_epoch_churn) + .ok_or(Error::ArithmeticOverflow( + "additional_epochs * per_epoch_churn", + ))?; + earliest_exit_epoch = earliest_exit_epoch.checked_add(additional_epochs).ok_or( + Error::ArithmeticOverflow("earliest_exit_epoch + additional_epochs"), + )?; + exit_balance_to_consume = exit_balance_to_consume + .checked_add(additional_churn) .ok_or(Error::ArithmeticOverflow( - "earliest_exit_epoch + additional_epochs", + "exit_balance_to_consume + additional_epochs * per_epoch_churn", ))?; - exit_balance_to_consume = exit_balance_to_consume - .checked_add(additional_churn) - .ok_or(Error::ArithmeticOverflow( - "exit_balance_to_consume + additional_epochs * per_epoch_churn", - ))?; - } - - *fields.exit_balance_to_consume_mut() = exit_balance_to_consume - .checked_sub(exit_balance) - .ok_or(Error::ArithmeticOverflow( - "exit_balance_to_consume - exit_balance", - ))?; - *fields.earliest_exit_epoch_mut() = earliest_exit_epoch; + } - Ok(earliest_exit_epoch) + let remaining = + exit_balance_to_consume + .checked_sub(exit_balance) + .ok_or(Error::ArithmeticOverflow( + "exit_balance_to_consume - exit_balance", + ))?; + self.exit_balance_to_consume = remaining; + self.earliest_exit_epoch = earliest_exit_epoch; + Ok(earliest_exit_epoch) + } } /// Advances electra's consolidation-queue cursor for a consolidation moving diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs index b9c6d4e2..cd7c70fe 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs @@ -24,7 +24,9 @@ use crate::beacon::helpers::accessors::get_current_epoch; use crate::beacon::helpers::altair::get_next_sync_committee; use crate::beacon::helpers::finality::is_in_inactivity_leak; use crate::beacon::helpers::math::saturating_sub; -use crate::beacon::helpers::participation::{EpochSummary, ParticipationTotals, RewardContext}; +use crate::beacon::helpers::participation::{ + EpochFlags, EpochSummary, ParticipationTotals, RewardContext, +}; use crate::beacon::preset; use crate::beacon::primitives::{Gwei, ValidatorIndex}; @@ -72,7 +74,10 @@ pub fn process_justification_and_finalization(state: &mut BeaconState) -> Result } /// Feeds `totals` to [`weigh_justification_and_finalization`]. -fn weigh_with_totals(state: &mut BeaconState, totals: &ParticipationTotals) -> Result<()> { +pub(super) fn weigh_with_totals( + state: &mut BeaconState, + totals: &ParticipationTotals, +) -> Result<()> { weigh_justification_and_finalization( state, totals.total_active_balance, @@ -129,22 +134,7 @@ fn update_inactivity_scores( len: score_count, })?; - let mut updated = *score; - if flags.participated(constants::TIMELY_TARGET_FLAG_INDEX) { - // `x -= min(1, x)`, written with `saturating_sub` so a - // already-zero score cannot underflow. - updated = saturating_sub(updated, 1); - } else { - // The specification treats a `uint64` overflow here as an invalid - // state rather than a wrapped one, so this is checked rather than - // left to release-mode wrapping. - updated = updated.checked_add(config.inactivity_score_bias).ok_or( - Error::ArithmeticOverflow("inactivity_scores[index] + INACTIVITY_SCORE_BIAS"), - )?; - } - if !leaking { - updated = saturating_sub(updated, config.inactivity_score_recovery_rate); - } + let updated = next_inactivity_score(*score, flags, leaking, config)?; if updated != *score { *score = updated; @@ -154,6 +144,39 @@ fn update_inactivity_scores( Ok(()) } +/// One eligible validator's post-step-2 inactivity score. +/// +/// Shared by the standalone step and the electra single pass, so both apply +/// the same rule and raise the same overflow error. `leaking` must be read +/// after justification. +pub(super) fn next_inactivity_score( + score: u64, + flags: EpochFlags, + leaking: bool, + config: &Config, +) -> Result { + let mut updated = score; + if flags.participated(constants::TIMELY_TARGET_FLAG_INDEX) { + // `x -= min(1, x)`, written with `saturating_sub` so a + // already-zero score cannot underflow. + updated = saturating_sub(updated, 1); + } else { + // The specification treats a `uint64` overflow here as an invalid + // state rather than a wrapped one, so this is checked rather than + // left to release-mode wrapping. + updated = + updated + .checked_add(config.inactivity_score_bias) + .ok_or(Error::ArithmeticOverflow( + "inactivity_scores[index] + INACTIVITY_SCORE_BIAS", + ))?; + } + if !leaking { + updated = saturating_sub(updated, config.inactivity_score_recovery_rate); + } + Ok(updated) +} + /// Applies the epoch's flag-index and inactivity deltas to every validator's /// balance. /// diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index f44d6f28..2dd81711 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -66,7 +66,43 @@ use crate::beacon::primitives::{ /// update, so a deposit credited this epoch is already reflected when /// effective balances round toward it). pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { - super::altair::process_participation_steps(state, config)?; + // Everything from justification through the effective-balance updates, + // in one pass over the registry where the state allows it. See + // `super::single_pass` for the fused path and [`process_unfused_steps`] + // for the specification-shaped one it is checked against. + super::single_pass::process_steps_through_effective_balances(state, config)?; + super::process_slashings_reset(state)?; + super::process_randao_mixes_reset(state)?; + super::capella::process_historical_summaries_update(state)?; + super::altair::process_participation_flag_updates(state)?; + super::altair::process_sync_committee_updates(state)?; + Ok(()) +} + +/// Steps 1-9 as the specification lists them, one function per step. +/// +/// The path [`super::single_pass`] replaces for registries it can fuse, and +/// falls back to for states it cannot (see there). Steps 1-3 run through the +/// altair driver's flat passes, or, with `reference_participation`, through +/// the specification-shaped code they replaced: the single pass's debug oracle +/// asks for that, so the fused path is checked against the original +/// formulation and not against its own building blocks. +pub(super) fn process_unfused_steps( + state: &mut BeaconState, + config: &Config, + reference_participation: bool, +) -> Result<()> { + #[cfg(any(test, debug_assertions))] + if reference_participation { + super::altair_reference::process_participation_steps(state, config)?; + } else { + super::altair::process_participation_steps(state, config)?; + } + #[cfg(not(any(test, debug_assertions)))] + { + let _ = reference_participation; + super::altair::process_participation_steps(state, config)?; + } // [Modified in Electra:EIP7251] process_registry_updates(state, config)?; // [Modified in Electra:EIP7251]: electra's own copy; see this module's @@ -79,13 +115,7 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { // [New in Electra:EIP7251] process_pending_consolidations(state, config)?; // [Modified in Electra:EIP7251] - process_effective_balance_updates(state)?; - super::process_slashings_reset(state)?; - super::process_randao_mixes_reset(state)?; - super::capella::process_historical_summaries_update(state)?; - super::altair::process_participation_flag_updates(state)?; - super::altair::process_sync_committee_updates(state)?; - Ok(()) + process_effective_balance_updates(state) } // --------------------------------------------------------------------------- @@ -94,12 +124,36 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { /// A validator's outcome for one run of [`process_registry_updates`]: at most /// one of the specification's `if`/`elif`/`elif` branches applies. -enum RegistryAction { +pub(super) enum RegistryAction { QueueForActivation, Eject, Activate, } +/// The one registry change `validator` gets this epoch, if any. +/// +/// The three branches are tested in the specification's order, and each +/// reads only the validator itself plus epoch-wide values, which is what lets +/// the single pass decide it without looking at any other validator. +pub(super) fn registry_action( + validator: &Validator, + current_epoch: Epoch, + finalized_epoch: Epoch, + ejection_balance: Gwei, +) -> Option { + if is_eligible_for_activation_queue(validator) { + Some(RegistryAction::QueueForActivation) + } else if is_active_validator(validator, current_epoch) + && validator.effective_balance <= ejection_balance + { + Some(RegistryAction::Eject) + } else if is_eligible_for_activation(validator, finalized_epoch) { + Some(RegistryAction::Activate) + } else { + None + } +} + /// Moves validators between activation-queue eligibility, ejection, and /// activation, replacing `super::registry::process_registry_updates` for this /// fork. @@ -134,17 +188,12 @@ pub fn process_registry_updates(state: &mut BeaconState, config: &Config) -> Res .validators() .iter() .map(|validator| { - if is_eligible_for_activation_queue(validator) { - Some(RegistryAction::QueueForActivation) - } else if is_active_validator(validator, current_epoch) - && validator.effective_balance <= config.ejection_balance - { - Some(RegistryAction::Eject) - } else if is_eligible_for_activation(validator, finalized_epoch) { - Some(RegistryAction::Activate) - } else { - None - } + registry_action( + validator, + current_epoch, + finalized_epoch, + config.ejection_balance, + ) }) .collect(); @@ -211,47 +260,14 @@ pub fn process_registry_updates(state: &mut BeaconState, config: &Config) -> Res /// `super::registry::process_slashings` does; the specification's own version /// takes no configuration either. pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<()> { - let epoch = get_current_epoch(state); - let total_balance = get_total_active_balance(state)?; - - let mut slashed_sum: Gwei = 0; - for &slashing in state.slashings().iter() { - slashed_sum = slashed_sum - .checked_add(slashing) - .ok_or(Error::ArithmeticOverflow("summing the slashings vector"))?; - } - let multiplier = preset::retuned::proportional_slashing_multiplier(state.fork_name()); - let scaled_slashings = slashed_sum - .checked_mul(multiplier) - .ok_or(Error::ArithmeticOverflow( - "scaling the summed slashings by the proportional multiplier", - ))?; - let adjusted_total_slashing_balance = scaled_slashings.min(total_balance); - - let increment = preset::EFFECTIVE_BALANCE_INCREMENT; - // `total_balance` (`get_total_active_balance`) sums every active - // validator's own increment-quantized effective balance and is floored at - // one whole increment, so it is always an exact multiple of `increment`: - // this can never divide by zero, and no remainder is lost to carry - // forward. - let total_increments = total_balance / increment; - let penalty_per_effective_balance_increment = - adjusted_total_slashing_balance / total_increments; - - let withdrawable_offset = (preset::EPOCHS_PER_SLASHINGS_VECTOR / 2) as Epoch; + let context = SlashingsContext::new(state, get_total_active_balance(state)?)?; // Collecting the penalties before applying any of them keeps this pass // reading a stable registry, the same reason // `super::registry::process_slashings` does. let mut penalties = Vec::new(); for (index, validator) in state.validators().iter().enumerate() { - if validator.slashed && epoch + withdrawable_offset == validator.withdrawable_epoch { - let effective_balance_increments = validator.effective_balance / increment; - let penalty = penalty_per_effective_balance_increment - .checked_mul(effective_balance_increments) - .ok_or(Error::ArithmeticOverflow( - "penalty_per_effective_balance_increment * effective_balance_increments", - ))?; + if let Some(penalty) = context.penalty(validator)? { penalties.push((index as ValidatorIndex, penalty)); } } @@ -263,6 +279,67 @@ pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<() Ok(()) } +/// The epoch-wide part of [`process_slashings`]: everything but the +/// per-validator effective balance. +pub(super) struct SlashingsContext { + epoch: Epoch, + penalty_per_effective_balance_increment: Gwei, +} + +impl SlashingsContext { + /// Sums and scales the slashings vector against `total_balance`, the + /// current epoch's `get_total_active_balance`. Fails where + /// [`process_slashings`] does, whether or not any validator is due. + pub(super) fn new(state: &BeaconState, total_balance: Gwei) -> Result { + let epoch = get_current_epoch(state); + + let mut slashed_sum: Gwei = 0; + for &slashing in state.slashings().iter() { + slashed_sum = slashed_sum + .checked_add(slashing) + .ok_or(Error::ArithmeticOverflow("summing the slashings vector"))?; + } + let multiplier = preset::retuned::proportional_slashing_multiplier(state.fork_name()); + let scaled_slashings = + slashed_sum + .checked_mul(multiplier) + .ok_or(Error::ArithmeticOverflow( + "scaling the summed slashings by the proportional multiplier", + ))?; + let adjusted_total_slashing_balance = scaled_slashings.min(total_balance); + + // `total_balance` (`get_total_active_balance`) sums every active + // validator's own increment-quantized effective balance and is floored + // at one whole increment, so it is always an exact multiple of + // `increment`: this can never divide by zero, and no remainder is lost + // to carry forward. + let total_increments = total_balance / preset::EFFECTIVE_BALANCE_INCREMENT; + Ok(Self { + epoch, + penalty_per_effective_balance_increment: adjusted_total_slashing_balance + / total_increments, + }) + } + + /// The penalty due from `validator` this epoch, or `None` when its + /// slashing does not fall due now. + pub(super) fn penalty(&self, validator: &Validator) -> Result> { + let withdrawable_offset = (preset::EPOCHS_PER_SLASHINGS_VECTOR / 2) as Epoch; + if !(validator.slashed && self.epoch + withdrawable_offset == validator.withdrawable_epoch) + { + return Ok(None); + } + let effective_balance_increments = + validator.effective_balance / preset::EFFECTIVE_BALANCE_INCREMENT; + self.penalty_per_effective_balance_increment + .checked_mul(effective_balance_increments) + .map(Some) + .ok_or(Error::ArithmeticOverflow( + "penalty_per_effective_balance_increment * effective_balance_increments", + )) + } +} + // --------------------------------------------------------------------------- // Pending deposits // --------------------------------------------------------------------------- @@ -436,7 +513,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res /// this, still builds a new validator phase0's way; see /// [`add_validator_from_pending_deposit`]'s doc for exactly how reusing it /// would go wrong here. -fn apply_pending_deposit( +pub(super) fn apply_pending_deposit( state: &mut BeaconState, deposit: &electra::PendingDeposit, config: &Config, @@ -634,11 +711,6 @@ pub fn process_pending_consolidations(state: &mut BeaconState, _config: &Config) /// algebraically equivalent but rounds differently, and only the /// specification's own order reproduces its integer rounding. pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> { - const HYSTERESIS_INCREMENT: Gwei = - preset::EFFECTIVE_BALANCE_INCREMENT / preset::HYSTERESIS_QUOTIENT; - const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; - const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; - // Two passes for the same reason `super::process_effective_balance_updates` // needs them: `state` is an enum over per-fork structs, so there is no // way to hold `validators` mutably while also reading `balances`, or @@ -653,12 +725,7 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> .zip(state.balances().iter()) .enumerate() { - if balance + DOWNWARD_THRESHOLD < validator.effective_balance - || validator.effective_balance + UPWARD_THRESHOLD < balance - { - let max_effective_balance = get_max_effective_balance(validator); - let effective = (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT) - .min(max_effective_balance); + if let Some(effective) = updated_effective_balance(validator, balance) { updates.push((index, effective)); } } @@ -670,6 +737,24 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> Ok(()) } +/// The effective balance `validator` moves to given its `balance`, or `None` +/// when the balance is still inside the hysteresis band. +pub(super) fn updated_effective_balance(validator: &Validator, balance: Gwei) -> Option { + const HYSTERESIS_INCREMENT: Gwei = + preset::EFFECTIVE_BALANCE_INCREMENT / preset::HYSTERESIS_QUOTIENT; + const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; + const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; + + if balance + DOWNWARD_THRESHOLD < validator.effective_balance + || validator.effective_balance + UPWARD_THRESHOLD < balance + { + let max_effective_balance = get_max_effective_balance(validator); + Some((balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT).min(max_effective_balance)) + } else { + None + } +} + // --------------------------------------------------------------------------- // Fork projection // --------------------------------------------------------------------------- @@ -684,7 +769,7 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> /// though its shape, an electra-or-fulu match, is identical: fulu keeps every /// field this covers unchanged (see `crate::beacon::helpers::electra`'s own module /// doc for why that module's projection accepts fulu too). -enum PendingQueueFields<'a> { +pub(super) enum PendingQueueFields<'a> { Electra(&'a mut electra::BeaconState), Fulu(&'a mut fulu::BeaconState), } @@ -694,7 +779,7 @@ impl<'a> PendingQueueFields<'a> { /// from crediting deposits off `Eth1Data` votes to crediting them off /// `DepositRequest`s directly, read by [`process_pending_deposits`] to /// know whether any eth1-bridge deposit is still outstanding. - fn deposit_requests_start_index(&self) -> u64 { + pub(super) fn deposit_requests_start_index(&self) -> u64 { match self { PendingQueueFields::Electra(state) => state.deposit_requests_start_index, PendingQueueFields::Fulu(state) => state.deposit_requests_start_index, @@ -702,14 +787,14 @@ impl<'a> PendingQueueFields<'a> { } /// How much of this epoch's deposit balance churn limit remains unused. - fn deposit_balance_to_consume(&self) -> Gwei { + pub(super) fn deposit_balance_to_consume(&self) -> Gwei { match self { PendingQueueFields::Electra(state) => state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => state.deposit_balance_to_consume, } } - fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { + pub(super) fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { match self { PendingQueueFields::Electra(state) => &mut state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => &mut state.deposit_balance_to_consume, @@ -717,7 +802,7 @@ impl<'a> PendingQueueFields<'a> { } /// Deposits known but not yet credited to the validator registry. - fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { + pub(super) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { match self { PendingQueueFields::Electra(state) => &mut state.pending_deposits, PendingQueueFields::Fulu(state) => &mut state.pending_deposits, @@ -725,7 +810,7 @@ impl<'a> PendingQueueFields<'a> { } /// Consolidations known but not yet applied. - fn pending_consolidations_mut(&mut self) -> &mut electra::PendingConsolidations { + pub(super) fn pending_consolidations_mut(&mut self) -> &mut electra::PendingConsolidations { match self { PendingQueueFields::Electra(state) => &mut state.pending_consolidations, PendingQueueFields::Fulu(state) => &mut state.pending_consolidations, @@ -756,7 +841,7 @@ impl<'a> PendingQueueFields<'a> { /// The electra-or-fulu state, mutably, through [`PendingQueueFields`]. See /// its own doc for why this is a second projection rather than a call into /// [`crate::beacon::helpers::electra::electra_state`]. -fn pending_queue_fields<'a>( +pub(super) fn pending_queue_fields<'a>( state: &'a mut BeaconState, function: &'static str, ) -> Result> { diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index 84d336a2..531c5740 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -25,6 +25,9 @@ pub mod justification; mod participation_equivalence; pub mod registry; pub mod rewards; +mod single_pass; +#[cfg(test)] +mod single_pass_equivalence; use crate::beacon::containers::phase0::PendingAttestation; use crate::beacon::containers::{BeaconState, HistoricalBatch}; diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs new file mode 100644 index 00000000..ef927771 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs @@ -0,0 +1,678 @@ +//! Electra's epoch steps 1-9 over one walk of the validator registry. +//! +//! Run step by step, the specification reads the registry once per step: +//! inactivity updates, rewards, registry updates, slashings, pending deposits +//! and effective-balance updates each walk it, and several of them touch every +//! active validator through a tree descent. At mainnet scale (millions of +//! entries) that is the bulk of an epoch-start block. Here one loop visits +//! each validator once, in index order, on a local copy of it and of its +//! balance: +//! +//! ```text +//! inactivity score -> rewards and penalties -> registry update +//! -> slashing penalty -> this validator's planned deposits +//! -> effective-balance update (unless deferred) +//! ``` +//! +//! then the changes are written back. That is the specification's order for +//! every validator, and steps only ever read the validator being processed +//! plus values fixed before the loop, which is what makes the fusion exact: +//! +//! - Justification runs first, unchanged, on the totals of +//! [`EpochSummary`]; the reward constants are built after it. +//! - The total active balance is the summary's. It equals +//! `get_total_active_balance` from step 1 through step 8: exits and +//! activations scheduled by registry updates take effect no earlier than +//! `compute_activation_exit_epoch`, and effective balances only move in +//! step 9. Slashings and every churn limit reuse it. +//! - Ejections advance the exit-churn cursor in index order, so the loop +//! carries a local [`ExitChurnCursor`] and writes it back once. +//! - Pending deposits are decided before the loop ([`DepositPlan`]), since +//! the walk down the queue depends on the registry only through each +//! deposit's validator: existing validators are topped up inside the loop, +//! deposits for pubkeys the registry has never seen are applied after it +//! (they append validators), in queue order. +//! - Pending consolidations move balance between two validators and read the +//! source's effective balance, so they run after the loop through the +//! ordinary step, and the effective-balance update of every validator a +//! consolidation names is deferred until they are done. +//! +//! What cannot be fused falls back to the step-by-step path +//! ([`super::electra::process_unfused_steps`]): the genesis epoch (no +//! participation to account for), and any state whose registry-sized lists +//! disagree in length, where the specification's per-step errors depend on +//! which step reads which list. +//! +//! In debug builds (`release-fast` keeps debug assertions) a registry small +//! enough to afford it is also run through the specification-shaped steps on +//! a clone, and the two full states must hash equal. + +use std::borrow::Cow; +use std::collections::BTreeMap; + +use crate::beacon::config::Config; +use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; +use crate::beacon::containers::shared::Validator; +use crate::beacon::containers::{BeaconState, electra}; +use crate::beacon::error::{Error, Result}; +use crate::beacon::helpers::accessors::get_current_epoch; +use crate::beacon::helpers::electra::{ExitChurnCursor, activation_exit_churn_limit_for}; +use crate::beacon::helpers::misc::{compute_activation_exit_epoch, compute_start_slot_at_epoch}; +use crate::beacon::helpers::participation::{EpochSummary, RewardContext}; +use crate::beacon::preset; +use crate::beacon::primitives::{BlsPubkey, Epoch, Gwei, ValidatorIndex}; + +use super::altair::{next_inactivity_score, weigh_with_totals}; +use super::electra::{ + RegistryAction, SlashingsContext, apply_pending_deposit, pending_queue_fields, + process_pending_consolidations, process_unfused_steps, registry_action, + updated_effective_balance, +}; + +/// Registries above this size skip the debug oracle: it is the slow path this +/// module replaces. +#[cfg(debug_assertions)] +const ORACLE_LIMIT: usize = 4096; + +/// Electra and fulu steps 1-9: justification, inactivity updates, rewards and +/// penalties, registry updates, slashings, eth1 data reset, pending deposits, +/// pending consolidations and effective-balance updates. +pub(super) fn process_steps_through_effective_balances( + state: &mut BeaconState, + config: &Config, +) -> Result<()> { + #[cfg(debug_assertions)] + { + if state.validators().len() <= ORACLE_LIMIT { + let mut expected = state.clone(); + let expected_result = process_unfused_steps(&mut expected, config, true); + let result = dispatch(state, config); + assert_eq!( + result.is_ok(), + expected_result.is_ok(), + "single pass disagrees with the unfused steps: {result:?} vs {expected_result:?}" + ); + if result.is_ok() { + assert_same_state(state, &mut expected); + } + return result; + } + } + dispatch(state, config) +} + +pub(super) fn dispatch(state: &mut BeaconState, config: &Config) -> Result<()> { + if can_fuse(state) { + fused(state, config) + } else { + process_unfused_steps(state, config, false) + } +} + +/// Whether the single pass reproduces the unfused steps on `state`. +/// +/// It does when every registry-sized list is exactly as long as the registry, +/// so no step can fail on a missing entry, and there is participation to +/// account for. +pub(super) fn can_fuse(state: &BeaconState) -> bool { + if !matches!(state, BeaconState::Electra(_) | BeaconState::Fulu(_)) { + return false; + } + if get_current_epoch(state) == constants::GENESIS_EPOCH { + return false; + } + let count = state.validators().len(); + let Ok((previous, current, scores)) = state.altair_validator_lists() else { + return false; + }; + state.balances().len() == count + && previous.len() == count + && current.len() == count + && scores.len() == count +} + +fn fused(state: &mut BeaconState, config: &Config) -> Result<()> { + let current_epoch = get_current_epoch(state); + + // Step 1, on the summary's totals; the reward constants read the + // finalized checkpoint it may move, so they come after. + let justifies = current_epoch > constants::GENESIS_EPOCH + 1; + let summary = EpochSummary::build(state, justifies)?; + if justifies { + weigh_with_totals(state, summary.totals())?; + } + let total_active_balance = summary.totals().total_active_balance; + let rewards = RewardContext::new(state, summary.totals())?; + + let finalized_epoch = state.finalized_checkpoint().epoch; + let activation_epoch = compute_activation_exit_epoch(current_epoch); + let per_epoch_churn = activation_exit_churn_limit_for(total_active_balance, config); + let slashings = SlashingsContext::new(state, total_active_balance)?; + let mut exit_cursor = ExitChurnCursor::read(state)?; + let exit_cursor_before = exit_cursor; + let mut plan = DepositPlan::new( + state, + config, + per_epoch_churn, + current_epoch, + finalized_epoch, + )?; + let deferred = consolidation_participants(state)?; + + // The loop only reads the state; every change is collected and written + // after it. + let mut score_changes: Vec<(usize, u64)> = Vec::new(); + let mut validator_changes: Vec<(usize, Validator)> = Vec::new(); + let mut balance_changes: Vec<(usize, Gwei)> = Vec::new(); + { + let validators = state.validators(); + let balances = state.balances(); + let (_, _, scores) = state.altair_validator_lists()?; + let mut top_ups = plan.top_ups.iter().peekable(); + let mut deferred_cursor = deferred.iter().peekable(); + + for (index, (((validator, &balance), &score), (flags, _))) in validators + .iter() + .zip(balances.iter()) + .zip(scores.iter()) + .zip(summary.iter()) + .enumerate() + { + let mut new_balance = balance; + let mut new_score = score; + + // Steps 2 and 3. + if flags.is_eligible() { + new_score = next_inactivity_score(score, flags, rewards.is_leaking(), config)?; + let deltas = + rewards.deltas(flags, validator.effective_balance, || Ok(new_score), config)?; + new_balance = deltas.apply(new_balance); + } + + // Step 4. + let mut current: Cow<'_, Validator> = Cow::Borrowed(validator); + match registry_action( + validator, + current_epoch, + finalized_epoch, + config.ejection_balance, + ) { + Some(RegistryAction::QueueForActivation) => { + current.to_mut().activation_eligibility_epoch = current_epoch + 1; + } + Some(RegistryAction::Eject) if validator.exit_epoch == FAR_FUTURE_EPOCH => { + let exit_epoch = exit_cursor.advance( + validator.effective_balance, + per_epoch_churn, + current_epoch, + )?; + // Checked for the reason `initiate_validator_exit` gives. + let withdrawable = exit_epoch + .checked_add(config.min_validator_withdrawability_delay) + .ok_or(Error::ArithmeticOverflow( + "exit_queue_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY", + ))?; + let ejected = current.to_mut(); + ejected.exit_epoch = exit_epoch; + ejected.withdrawable_epoch = withdrawable; + } + Some(RegistryAction::Activate) => { + current.to_mut().activation_epoch = activation_epoch; + } + // Already exiting: `initiate_validator_exit` does nothing. + Some(RegistryAction::Eject) | None => {} + } + + // Step 5, on the validator as step 4 left it. + if let Some(penalty) = slashings.penalty(¤t)? { + new_balance = new_balance.saturating_sub(penalty); + } + + // Step 7, for deposits naming this validator. + if let Some(&&(_, amount)) = top_ups.peek().filter(|&&&(at, _)| at == index) { + new_balance = new_balance.saturating_add(amount); + top_ups.next(); + } + + // Step 9, unless a pending consolidation may still move this + // validator's balance or read its effective balance. + while deferred_cursor.next_if(|&&at| at < index).is_some() {} + let is_deferred = deferred_cursor.next_if(|&&at| at == index).is_some(); + if !is_deferred + && let Some(effective) = updated_effective_balance(¤t, new_balance) + && effective != current.effective_balance + { + current.to_mut().effective_balance = effective; + } + + if let Cow::Owned(validator) = current { + validator_changes.push((index, validator)); + } + if new_balance != balance { + balance_changes.push((index, new_balance)); + } + if new_score != score { + score_changes.push((index, new_score)); + } + } + } + drop(summary); + + write_back( + state, + validator_changes, + balance_changes, + score_changes, + exit_cursor, + exit_cursor_before, + )?; + + // Step 6 (eth1 votes are untouched by everything above and below). + super::process_eth1_data_reset(state)?; + + // Step 7: the queue itself, then the deposits that create validators. + let registry_len = state.validators().len(); + plan.finish(state, config)?; + + // Step 8, then step 9 for what was deferred, and for the validators the + // deposits just created. + process_pending_consolidations(state, config)?; + let new_registry_len = state.validators().len(); + let mut effective_updates = Vec::new(); + let patched = deferred + .iter() + .copied() + .filter(|&index| index < registry_len) + .chain(registry_len..new_registry_len); + for index in patched { + let validator = state.validator(index as ValidatorIndex)?; + let balance = state.balance(index as ValidatorIndex)?; + if let Some(effective) = updated_effective_balance(validator, balance) { + effective_updates.push((index, effective)); + } + } + for (index, effective) in effective_updates { + state + .validator_mut(index as ValidatorIndex)? + .effective_balance = effective; + } + Ok(()) +} + +/// Stores what the loop decided. +fn write_back( + state: &mut BeaconState, + validator_changes: Vec<(usize, Validator)>, + balance_changes: Vec<(usize, Gwei)>, + score_changes: Vec<(usize, u64)>, + exit_cursor: ExitChurnCursor, + exit_cursor_before: ExitChurnCursor, +) -> Result<()> { + let validators = state.validators_mut(); + for (index, validator) in validator_changes { + *validators + .get_mut(index) + .ok_or(Error::UnknownValidator(index as ValidatorIndex))? = validator; + } + let balances = state.balances_mut(); + for (index, balance) in balance_changes { + *balances + .get_mut(index) + .ok_or(Error::UnknownValidator(index as ValidatorIndex))? = balance; + } + let (_, _, scores) = state.altair_validator_lists_mut()?; + let score_count = scores.len(); + for (index, score) in score_changes { + *scores.get_mut(index).ok_or(Error::IndexOutOfBounds { + index, + len: score_count, + })? = score; + } + if exit_cursor != exit_cursor_before { + exit_cursor.write(state)?; + } + Ok(()) +} + +/// The validators named by any pending consolidation, sorted and deduplicated. +/// +/// A superset of the ones the consolidation step will actually touch (it +/// stops at the first source that is not withdrawable yet), which is safe: +/// deferring a validator's effective-balance update to after that step and +/// then doing it on its final balance is the same as doing it in the loop for +/// a validator nothing touched. +fn consolidation_participants(state: &mut BeaconState) -> Result> { + let mut fields = pending_queue_fields(state, "single-pass epoch processing")?; + let mut participants: Vec = fields + .pending_consolidations_mut() + .iter() + .flat_map(|consolidation| { + [ + consolidation.source_index as usize, + consolidation.target_index as usize, + ] + }) + .collect(); + participants.sort_unstable(); + participants.dedup(); + Ok(participants) +} + +/// What [`super::electra::process_pending_deposits`] would do, decided before +/// the registry loop. +struct DepositPlan { + /// Total top-up per existing validator, ascending by index. Applied in the + /// loop, at the point of the specification's deposit step. + top_ups: Vec<(usize, Gwei)>, + /// Deposits whose pubkey the registry has not got, in queue order. They + /// are applied after the loop through [`apply_pending_deposit`], which + /// creates the validator (if the signature holds) and tops it up on later + /// deposits for the same pubkey. + new_pubkey_deposits: Vec, + /// The queue as it stands after this epoch: what was not reached, then + /// what was postponed. + remaining: Vec, + deposit_balance_to_consume: Gwei, +} + +/// How a reached deposit was resolved. +enum DepositKind { + /// Credited to a validator already in the registry. + TopUp, + /// Named a pubkey the registry lacks. + NewPubkey, + /// Named an exiting validator: moved to the back of the queue. + Postpone, +} + +impl DepositPlan { + /// Walks the queue exactly as `process_pending_deposits` does, without + /// applying anything: same stopping rules, same churn accounting, same + /// errors. + /// + /// A validator counts as exited when it already has an exit epoch or when + /// registry updates are about to eject it (the ejection sets one), and as + /// withdrawn by its withdrawable epoch before registry updates: an + /// ejection that changes anything moves that epoch to + /// `compute_activation_exit_epoch` or later, which is never before the + /// next epoch. + fn new( + state: &mut BeaconState, + config: &Config, + per_epoch_churn: Gwei, + current_epoch: Epoch, + finalized_epoch: Epoch, + ) -> Result { + let next_epoch = current_epoch + 1; + let finalized_slot = compute_start_slot_at_epoch(finalized_epoch); + let eth1_deposit_index = state.eth1_deposit_index(); + + let (deposit_requests_start_index, available_for_processing, mut deposits) = { + let mut fields = pending_queue_fields(state, "single-pass epoch processing")?; + let available_for_processing = fields + .deposit_balance_to_consume() + .checked_add(per_epoch_churn) + .ok_or(Error::ArithmeticOverflow( + "deposit_balance_to_consume + get_activation_exit_churn_limit", + ))?; + let deposits: Vec = + core::mem::take(fields.pending_deposits_mut()).into_inner(); + ( + fields.deposit_requests_start_index(), + available_for_processing, + deposits, + ) + }; + + // The ordering gates and the per-epoch cap read nothing from the + // registry, so the deposits the walk can reach are known up front. + let reachable = deposits + .iter() + .take(preset::MAX_PENDING_DEPOSITS_PER_EPOCH as usize) + .take_while(|deposit| { + !(deposit.slot > constants::GENESIS_SLOT + && eth1_deposit_index < deposit_requests_start_index) + && deposit.slot <= finalized_slot + }) + .count(); + let found = first_indices_of( + state, + deposits[..reachable].iter().map(|deposit| deposit.pubkey), + ); + + let mut processed_amount: Gwei = 0; + let mut is_churn_limit_reached = false; + let mut kinds: Vec = Vec::with_capacity(reachable); + let mut top_ups: BTreeMap = BTreeMap::new(); + let mut credit = |index: usize, amount: Gwei| { + let total = top_ups.entry(index).or_default(); + *total = total.saturating_add(amount); + }; + + for (deposit, existing) in deposits[..reachable].iter().zip(found) { + let (is_validator_exited, is_validator_withdrawn) = match existing { + Some(index) => { + let validator = state.validator(index as ValidatorIndex)?; + let ejected_now = validator.exit_epoch == FAR_FUTURE_EPOCH + && matches!( + registry_action( + validator, + current_epoch, + finalized_epoch, + config.ejection_balance, + ), + Some(RegistryAction::Eject) + ); + ( + validator.exit_epoch < FAR_FUTURE_EPOCH || ejected_now, + !ejected_now && validator.withdrawable_epoch < next_epoch, + ) + } + None => (false, false), + }; + + if is_validator_withdrawn { + let index = existing.expect("a withdrawn validator is in the registry"); + credit(index, deposit.amount); + kinds.push(DepositKind::TopUp); + } else if is_validator_exited { + kinds.push(DepositKind::Postpone); + } else { + match processed_amount.checked_add(deposit.amount) { + Some(sum) if sum <= available_for_processing => { + processed_amount = sum; + match existing { + Some(index) => { + credit(index, deposit.amount); + kinds.push(DepositKind::TopUp); + } + None => kinds.push(DepositKind::NewPubkey), + } + } + _ => { + is_churn_limit_reached = true; + break; + } + } + } + } + + let next_deposit_index = kinds.len(); + let tail = deposits.split_off(next_deposit_index); + let mut new_pubkey_deposits = Vec::new(); + let mut remaining = tail; + for (deposit, kind) in deposits.into_iter().zip(kinds) { + match kind { + DepositKind::TopUp => {} + DepositKind::NewPubkey => new_pubkey_deposits.push(deposit), + DepositKind::Postpone => remaining.push(deposit), + } + } + + let deposit_balance_to_consume = if is_churn_limit_reached { + available_for_processing + .checked_sub(processed_amount) + .ok_or(Error::ArithmeticOverflow( + "available_for_processing - processed_amount", + ))? + } else { + 0 + }; + + Ok(Self { + top_ups: top_ups.into_iter().collect(), + new_pubkey_deposits, + remaining, + deposit_balance_to_consume, + }) + } + + /// Writes the queue and the leftover churn back, then applies the + /// deposits that create validators, in queue order. + fn finish(&mut self, state: &mut BeaconState, config: &Config) -> Result<()> { + let remaining = core::mem::take(&mut self.remaining); + let mut fields = pending_queue_fields(state, "single-pass epoch processing")?; + *fields.pending_deposits_mut() = electra::PendingDeposits::try_from(remaining)?; + *fields.deposit_balance_to_consume_mut() = self.deposit_balance_to_consume; + + for deposit in core::mem::take(&mut self.new_pubkey_deposits) { + apply_pending_deposit(state, &deposit, config)?; + } + Ok(()) + } +} + +/// For each pubkey, in order, the first registry index holding it. +/// +/// One walk of the registry for all of them, instead of one per deposit. The +/// first-match rule is `process_pending_deposits`'s, and matters when a +/// crafted registry repeats a key. +fn first_indices_of( + state: &BeaconState, + pubkeys: impl Iterator, +) -> Vec> { + let wanted: Vec = pubkeys.collect(); + let mut found: Vec> = vec![None; wanted.len()]; + if wanted.is_empty() { + return found; + } + // The first eight bytes of a key are as good as random, so most + // validators are rejected by one integer comparison per wanted key. + let prefix = + |pubkey: &BlsPubkey| u64::from_le_bytes(pubkey.0[..8].try_into().expect("8 <= 48")); + let prefixes: Vec = wanted.iter().map(prefix).collect(); + let mut missing = wanted.len(); + for (index, validator) in state.validators().iter().enumerate() { + let validator_prefix = prefix(&validator.pubkey); + for (slot, wanted_prefix) in prefixes.iter().enumerate() { + if found[slot].is_none() + && *wanted_prefix == validator_prefix + && wanted[slot] == validator.pubkey + { + found[slot] = Some(index); + missing -= 1; + } + } + if missing == 0 { + break; + } + } + found +} + +/// Panics, naming what differs, unless the single pass and the unfused steps +/// left identical states. +#[cfg(any(test, debug_assertions))] +fn assert_same_state(state: &mut BeaconState, expected: &mut BeaconState) { + state.apply_pending_mutations(); + expected.apply_pending_mutations(); + if state.hash_tree_root() == expected.hash_tree_root() { + return; + } + panic!( + "single pass diverged from the unfused steps in: {}", + differing_fields(state, expected).join(", ") + ); +} + +/// The touched fields that differ, for a failing oracle to report. +#[cfg(any(test, debug_assertions))] +pub(super) fn differing_fields(state: &mut BeaconState, expected: &mut BeaconState) -> Vec { + let mut differences = Vec::new(); + let mut note = |field: &str, differs: bool| { + if differs { + differences.push(field.to_string()); + } + }; + + let describe_at = |what: &str, left: Vec, right: Vec| { + if left.len() != right.len() { + return Some(format!("{what} length {} vs {}", left.len(), right.len())); + } + left.iter() + .zip(&right) + .position(|(a, b)| a != b) + .map(|index| format!("{what}[{index}]: {} vs {}", left[index], right[index])) + }; + let debug_all = |items: &mut dyn Iterator| items.collect::>(); + + let left = debug_all(&mut state.validators().iter().map(|v| format!("{v:?}"))); + let right = debug_all(&mut expected.validators().iter().map(|v| format!("{v:?}"))); + if let Some(text) = describe_at("validators", left, right) { + note(&text, true); + } + + let left = debug_all(&mut state.balances().iter().map(|v| v.to_string())); + let right = debug_all(&mut expected.balances().iter().map(|v| v.to_string())); + if let Some(text) = describe_at("balances", left, right) { + note(&text, true); + } + + if let (Ok((_, _, left)), Ok((_, _, right))) = ( + state.altair_validator_lists(), + expected.altair_validator_lists(), + ) { + note("inactivity_scores", left != right); + } + note( + "finalized_checkpoint", + state.finalized_checkpoint() != expected.finalized_checkpoint(), + ); + note( + "current_justified_checkpoint", + state.current_justified_checkpoint() != expected.current_justified_checkpoint(), + ); + note( + "previous_justified_checkpoint", + state.previous_justified_checkpoint() != expected.previous_justified_checkpoint(), + ); + note( + "justification_bits", + state.justification_bits() != expected.justification_bits(), + ); + note( + "exit_churn_cursor", + ExitChurnCursor::read(state).ok() != ExitChurnCursor::read(expected).ok(), + ); + if let (Ok(mut left), Ok(mut right)) = ( + pending_queue_fields(state, "oracle"), + pending_queue_fields(expected, "oracle"), + ) { + note( + "pending_deposits", + left.pending_deposits_mut() != right.pending_deposits_mut(), + ); + note( + "deposit_balance_to_consume", + left.deposit_balance_to_consume() != right.deposit_balance_to_consume(), + ); + note( + "pending_consolidations", + left.pending_consolidations_mut() != right.pending_consolidations_mut(), + ); + } + if differences.is_empty() { + differences.push("a field outside the ones compared (eth1 votes, resets)".to_string()); + } + differences +} diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs new file mode 100644 index 00000000..c2111915 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs @@ -0,0 +1,553 @@ +//! Randomized equivalence of electra's fused epoch pass with the step-by-step +//! path it replaces. +//! +//! Each case builds an electra or fulu state at some epoch and runs steps 1-9 +//! twice: through [`single_pass`]'s fused driver and through the +//! specification-shaped steps (with the reference participation code). Both +//! must succeed or both fail, and on success the full states must hash equal. +//! +//! The generator crafts what live chains rarely hold together: several +//! ejections in one epoch (so the exit-churn cursor moves), validators waiting +//! for the activation queue and for activation, slashed validators due and not +//! due for their penalty, a non-zero slashings vector, balances on both sides +//! of the hysteresis thresholds, and pending queues that hit every branch: +//! top-ups of active, exiting and withdrawn validators, a validator about to be +//! ejected, new pubkeys with valid and invalid signatures (twice for one +//! pubkey), an exhausted churn budget, unfinalized slots, and consolidations +//! that are processed, dropped as slashed, or stop the queue. + +use super::electra; +use super::single_pass::{can_fuse, differing_fields, dispatch}; +use crate::beacon::config::Config; +use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; +use crate::beacon::containers::shared::{Checkpoint, DepositMessage}; +use crate::beacon::containers::{BeaconState, electra as electra_containers}; +use crate::beacon::fork::ForkName; +use crate::beacon::helpers::electra::ExitChurnCursor; +use crate::beacon::helpers::misc::{ + compute_deposit_domain, compute_signing_root, compute_start_slot_at_epoch, +}; +use crate::beacon::helpers::test_state::{secret_key_for, sign_for, with_signing_validators_at}; +use crate::beacon::preset; +use crate::beacon::primitives::{ + BlsPubkey, BlsSignature, Bytes32, Epoch, Gwei, HashTreeRoot as _, Root, ValidatorIndex, +}; + +/// Small deterministic generator, so a failing case reproduces from its seed +/// without a new dependency. +struct SplitMix64(u64); + +impl SplitMix64 { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform in `0..bound`. + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + /// True with probability `percent` in a hundred. + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } + + fn pick(&mut self, items: &[T]) -> T { + items[self.below(items.len() as u64) as usize] + } +} + +const INCREMENT: Gwei = preset::EFFECTIVE_BALANCE_INCREMENT; +const ETH: Gwei = 1_000_000_000; + +/// Registry sizes the generator draws from, and how many cases each fork gets. +const SIZES: [usize; 3] = [40, 120, 300]; +const CASES_PER_FORK: u64 = 400; + +/// A signed deposit for the key with index `key` (kept clear of the registry's +/// own keys), or an unsigned one when `valid` is false. +fn deposit_for_key( + key: usize, + amount: Gwei, + slot: u64, + valid: bool, + config: &Config, +) -> electra_containers::PendingDeposit { + let pubkey = BlsPubkey(secret_key_for(key).sk_to_pk().to_bytes()); + let mut withdrawal_credentials = Bytes32::ZERO; + withdrawal_credentials.0[0] = constants::COMPOUNDING_WITHDRAWAL_PREFIX; + let message = DepositMessage { + pubkey, + withdrawal_credentials, + amount, + }; + let domain = compute_deposit_domain(config.genesis_fork_version); + let signing_root = compute_signing_root(message.hash_tree_root(), domain); + let signature = if valid { + sign_for(key, signing_root) + } else { + BlsSignature::default() + }; + electra_containers::PendingDeposit { + pubkey, + withdrawal_credentials, + amount, + signature, + slot, + } +} + +/// A random state derived from `base` (a registry of signing validators). +fn random_state(rng: &mut SplitMix64, base: &BeaconState, config: &Config) -> BeaconState { + let mut state = base.clone(); + let count = state.validators().len(); + + let epoch: Epoch = match rng.below(10) { + // Genesis: the fused pass declines and the steps run one by one. + 0 => 0, + 1 => 1, + 2 => 2, + // Far enough past the finalized checkpoint to leak. + 3 => preset::MIN_EPOCHS_TO_INACTIVITY_PENALTY + 3 + rng.below(4), + _ => 3 + rng.below(40), + }; + let offset = if rng.chance(90) { + 1 + rng.below(preset::SLOTS_PER_EPOCH - 1) + } else { + 0 + }; + *state.slot_mut() = epoch * preset::SLOTS_PER_EPOCH + offset; + let next_epoch = epoch + 1; + + let slashings_offset = (preset::EPOCHS_PER_SLASHINGS_VECTOR / 2) as Epoch; + let hysteresis_down = + INCREMENT / preset::HYSTERESIS_QUOTIENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; + let hysteresis_up = + INCREMENT / preset::HYSTERESIS_QUOTIENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; + let participation_density = rng.pick(&[0, 30, 70, 95, 100]); + let near = |rng: &mut SplitMix64| (epoch + rng.below(5)).saturating_sub(2); + + // Finality first: the activation branch reads the finalized epoch. + let checkpoint = |epoch: u64| Checkpoint { + epoch, + root: Root::ZERO, + }; + *state.previous_justified_checkpoint_mut() = checkpoint(epoch.saturating_sub(rng.below(4))); + *state.current_justified_checkpoint_mut() = checkpoint(epoch.saturating_sub(rng.below(3))); + let finalized_epoch = if rng.chance(30) { + 0 + } else { + epoch.saturating_sub(rng.below(3)) + }; + *state.finalized_checkpoint_mut() = checkpoint(finalized_epoch); + for bit in 0..constants::JUSTIFICATION_BITS_LENGTH { + let value = rng.chance(50); + state.justification_bits_mut().set(bit, value).unwrap(); + } + + let mut balances: Vec = Vec::with_capacity(count); + for index in 0..count { + let validator = state.validator_mut(index as ValidatorIndex).unwrap(); + if rng.chance(30) { + validator.withdrawal_credentials.0[0] = constants::COMPOUNDING_WITHDRAWAL_PREFIX; + } + validator.activation_eligibility_epoch = match rng.below(20) { + 0..=2 => FAR_FUTURE_EPOCH, + 3 => epoch, + _ => 0, + }; + validator.activation_epoch = match rng.below(20) { + 0..=2 => FAR_FUTURE_EPOCH, + 3 => near(rng), + _ => 0, + }; + validator.exit_epoch = match rng.below(20) { + 0 | 1 => near(rng), + 2 => near(rng) + 1, + _ => FAR_FUTURE_EPOCH, + }; + validator.slashed = rng.chance(8); + validator.withdrawable_epoch = if validator.slashed && rng.chance(60) { + (epoch + slashings_offset) + .saturating_add(rng.pick(&[0, 0, 0, 1])) + .saturating_sub(rng.pick(&[0, 0, 1])) + } else { + match rng.below(4) { + 0 => near(rng), + 1 => near(rng) + 1, + _ => FAR_FUTURE_EPOCH, + } + }; + validator.effective_balance = match rng.below(20) { + 0 => 0, + 1..=3 => config.ejection_balance, + 4 => config.ejection_balance - INCREMENT, + 5 => config.ejection_balance + INCREMENT, + 6..=7 => preset::MAX_EFFECTIVE_BALANCE_ELECTRA, + 8 => (1 + rng.below(preset::MAX_EFFECTIVE_BALANCE_ELECTRA / INCREMENT)) * INCREMENT, + 9 => rng.below(preset::MAX_EFFECTIVE_BALANCE_ELECTRA), + _ => preset::MIN_ACTIVATION_BALANCE, + }; + let effective = validator.effective_balance; + balances.push(match rng.below(14) { + 0 => 0, + 1 => rng.below(5_000), + 2 => effective.saturating_sub(hysteresis_down + 1), + 3 => effective.saturating_sub(hysteresis_down), + 4 => effective + hysteresis_up + 1, + 5 => effective + hysteresis_up, + 6 => effective + rng.below(3 * INCREMENT), + 7 if rng.chance(20) => u64::MAX - rng.below(3), + _ => effective, + }); + } + *state.balances_mut() = balances.try_into().unwrap(); + + // Participation lists and scores, all bit patterns. + let lists: Vec> = (0..2) + .map(|_| { + (0..count) + .map(|_| { + if rng.chance(participation_density) { + if rng.chance(50) { + 0b111 | (rng.below(32) as u8) << 3 + } else { + rng.below(256) as u8 + } + } else { + 0 + } + }) + .collect() + }) + .collect(); + let scores: Vec = (0..count) + .map(|_| match rng.below(10) { + 0..=2 => 0, + 3..=6 => rng.below(64), + _ => rng.below(1 << 20), + }) + .collect(); + let (previous, current, inactivity) = state.altair_validator_lists_mut().unwrap(); + *previous = lists[0].clone().try_into().unwrap(); + *current = lists[1].clone().try_into().unwrap(); + *inactivity = scores.try_into().unwrap(); + + // A few states the fused pass declines: a short list. + if rng.chance(3) { + let keep = rng.below(count as u64) as usize; + let mut balances = state.balances().to_vec(); + balances.truncate(keep); + *state.balances_mut() = balances.try_into().unwrap(); + } + + // The slashings vector. + if rng.chance(60) { + for _ in 0..rng.below(12) { + let at = rng.below(preset::EPOCHS_PER_SLASHINGS_VECTOR as u64) as usize; + state.slashings_mut()[at] = rng.below(400 * ETH); + } + if rng.chance(3) { + state.slashings_mut()[0] = u64::MAX; + state.slashings_mut()[1] = u64::MAX; + } + } + + // The exit-churn cursor. + ExitChurnCursor { + earliest_exit_epoch: (epoch + rng.below(10)).saturating_sub(2), + exit_balance_to_consume: rng.below(200 * ETH), + } + .write(&mut state) + .unwrap(); + + // Pending consolidations. + let mut consolidations = Vec::new(); + for _ in 0..rng.pick(&[0, 0, 1, 3, 6]) { + let source = rng.below(count as u64); + let target = if rng.chance(3) { + count as u64 + rng.below(3) + } else { + rng.below(count as u64) + }; + let validator = state.validator_mut(source).unwrap(); + match rng.below(4) { + // Processed. + 0 | 1 => { + validator.slashed = false; + validator.withdrawable_epoch = rng.pick(&[epoch, next_epoch, 0]); + } + // Dropped as slashed. + 2 => validator.slashed = true, + // Not withdrawable yet: stops the queue. + _ => { + validator.slashed = false; + validator.withdrawable_epoch = next_epoch + 1 + rng.below(3); + } + } + consolidations.push(electra_containers::PendingConsolidation { + source_index: source, + target_index: target, + }); + } + + // Pending deposits. + let finalized_slot = compute_start_slot_at_epoch(finalized_epoch); + let mut deposits: Vec = Vec::new(); + let mut fresh_keys: Vec = Vec::new(); + for _ in 0..rng.pick(&[0, 0, 1, 2, 4, 8, 17, 20]) { + let random_amount = INCREMENT * (1 + rng.below(150)); + let amount = rng.pick(&[ETH, 32 * ETH, random_amount, u64::MAX / 2]); + let slot = match rng.below(10) { + 0 => finalized_slot + 1 + rng.below(3), + 1..=2 if finalized_slot > 0 => 1 + rng.below(finalized_slot), + _ => constants::GENESIS_SLOT, + }; + let pick_validator = |rng: &mut SplitMix64| rng.below(count as u64); + let deposit = match rng.below(9) { + // An existing validator, whatever its state. + 0 | 1 => { + let index = pick_validator(rng); + let validator = state.validator(index).unwrap(); + electra_containers::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount, + signature: BlsSignature::default(), + slot, + } + } + // An exiting validator, not yet withdrawable. + 2 => { + let index = pick_validator(rng); + let validator = state.validator_mut(index).unwrap(); + validator.exit_epoch = epoch + 3; + validator.withdrawable_epoch = epoch + 10; + electra_containers::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount, + signature: BlsSignature::default(), + slot, + } + } + // A withdrawn validator. + 3 => { + let index = pick_validator(rng); + let validator = state.validator_mut(index).unwrap(); + validator.exit_epoch = 0; + validator.withdrawable_epoch = 0; + electra_containers::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount, + signature: BlsSignature::default(), + slot, + } + } + // A validator this very epoch ejects, sometimes with a + // withdrawable epoch that only looks withdrawn until it is. + 4 => { + let index = pick_validator(rng); + let validator = state.validator_mut(index).unwrap(); + validator.activation_epoch = 0; + validator.exit_epoch = FAR_FUTURE_EPOCH; + validator.effective_balance = config.ejection_balance; + validator.activation_eligibility_epoch = 0; + if rng.chance(50) { + validator.withdrawable_epoch = 0; + } + electra_containers::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount, + signature: BlsSignature::default(), + slot, + } + } + // A new pubkey, valid or not. + 5 | 6 => { + let key = 10_000 + fresh_keys.len(); + fresh_keys.push(key); + deposit_for_key(key, amount.min(600 * ETH), slot, rng.chance(70), config) + } + // Another deposit for a pubkey already queued. + _ => match fresh_keys.last() { + Some(&key) => { + deposit_for_key(key, ETH * (1 + rng.below(40)), slot, rng.chance(70), config) + } + None => deposit_for_key(10_000, amount.min(600 * ETH), slot, true, config), + }, + }; + deposits.push(deposit); + } + + match &mut state { + BeaconState::Electra(inner) => { + if rng.chance(10) { + inner.eth1_deposit_index = 3; + inner.deposit_requests_start_index = 5; + } else { + inner.eth1_deposit_index = 5; + inner.deposit_requests_start_index = 5; + } + inner.deposit_balance_to_consume = rng.pick(&[0, 0, 10 * ETH, 500 * ETH]); + inner.pending_deposits = deposits.try_into().unwrap(); + inner.pending_consolidations = consolidations.try_into().unwrap(); + } + BeaconState::Fulu(inner) => { + if rng.chance(10) { + inner.eth1_deposit_index = 3; + inner.deposit_requests_start_index = 5; + } else { + inner.eth1_deposit_index = 5; + inner.deposit_requests_start_index = 5; + } + inner.deposit_balance_to_consume = rng.pick(&[0, 0, 10 * ETH, 500 * ETH]); + inner.pending_deposits = deposits.try_into().unwrap(); + inner.pending_consolidations = consolidations.try_into().unwrap(); + } + _ => unreachable!("the bases are electra or fulu"), + } + if rng.chance(2) { + // Overflows `deposit_balance_to_consume + churn`. + match &mut state { + BeaconState::Electra(inner) => inner.deposit_balance_to_consume = u64::MAX - 5, + BeaconState::Fulu(inner) => inner.deposit_balance_to_consume = u64::MAX - 5, + _ => unreachable!(), + } + } + state +} + +#[derive(Default, Debug)] +struct Coverage { + cases: u32, + fused: u32, + both_failed: u32, + ejected: u32, + activated: u32, + queued: u32, + validators_created: u32, + deposits_consumed: u32, + consolidations_consumed: u32, + effective_balance_moved: u32, + scores_moved: u32, +} + +#[test] +fn fused_pass_matches_the_unfused_steps() { + let config = Config::mainnet(); + let mut coverage = Coverage::default(); + for (fork_number, fork) in [ForkName::Electra, ForkName::Fulu].into_iter().enumerate() { + let bases: Vec = SIZES + .iter() + .map(|&size| with_signing_validators_at(fork, size)) + .collect(); + for case in 0..CASES_PER_FORK { + let seed = (fork_number as u64) << 32 | case; + let mut rng = SplitMix64(seed); + let base = &bases[rng.below(bases.len() as u64) as usize]; + let state = random_state(&mut rng, base, &config); + + let mut fused = state.clone(); + let mut unfused = state.clone(); + let fused_result = dispatch(&mut fused, &config); + let unfused_result = electra::process_unfused_steps(&mut unfused, &config, true); + + coverage.cases += 1; + coverage.fused += can_fuse(&state) as u32; + assert_eq!( + fused_result.is_ok(), + unfused_result.is_ok(), + "outcome differs, seed {seed}: fused {fused_result:?} vs unfused {unfused_result:?}" + ); + if fused_result.is_err() { + coverage.both_failed += 1; + continue; + } + + fused.apply_pending_mutations(); + unfused.apply_pending_mutations(); + if fused.hash_tree_root() != unfused.hash_tree_root() { + panic!( + "states differ, seed {seed}: {}", + differing_fields(&mut fused, &mut unfused).join(", ") + ); + } + + let before: Vec<_> = state.validators().iter().collect(); + let after: Vec<_> = fused.validators().iter().collect(); + for (old, new) in before.iter().zip(after.iter()) { + coverage.ejected += (old.exit_epoch == FAR_FUTURE_EPOCH + && new.exit_epoch != FAR_FUTURE_EPOCH) + as u32; + coverage.activated += (old.activation_epoch == FAR_FUTURE_EPOCH + && new.activation_epoch != FAR_FUTURE_EPOCH) + as u32; + coverage.queued += (old.activation_eligibility_epoch == FAR_FUTURE_EPOCH + && new.activation_eligibility_epoch != FAR_FUTURE_EPOCH) + as u32; + coverage.effective_balance_moved += + (old.effective_balance != new.effective_balance) as u32; + } + coverage.validators_created += (after.len() > before.len()) as u32; + let queue_len = |state: &mut BeaconState| { + let mut state = state.clone(); + let mut fields = super::electra::pending_queue_fields(&mut state, "test").unwrap(); + ( + fields.pending_deposits_mut().len(), + fields.pending_consolidations_mut().len(), + ) + }; + let (deposits_before, consolidations_before) = queue_len(&mut state.clone()); + let (deposits_after, consolidations_after) = queue_len(&mut fused); + coverage.deposits_consumed += (deposits_after < deposits_before) as u32; + coverage.consolidations_consumed += + (consolidations_after < consolidations_before) as u32; + let (_, _, scores_before) = state.altair_validator_lists().unwrap(); + let (_, _, scores_after) = fused.altair_validator_lists().unwrap(); + coverage.scores_moved += (scores_before != scores_after) as u32; + } + } + + println!("{coverage:?}"); + let total = coverage.cases; + assert!( + coverage.fused * 10 > total * 8, + "too few fused cases: {coverage:?}" + ); + assert!( + coverage.both_failed * 5 < total, + "too many errors: {coverage:?}" + ); + assert!(coverage.ejected > 100, "{coverage:?}"); + assert!(coverage.activated > 100, "{coverage:?}"); + assert!(coverage.queued > 100, "{coverage:?}"); + assert!(coverage.validators_created > 40, "{coverage:?}"); + assert!(coverage.deposits_consumed > 100, "{coverage:?}"); + assert!(coverage.consolidations_consumed > 50, "{coverage:?}"); + assert!(coverage.effective_balance_moved > 500, "{coverage:?}"); + assert!(coverage.scores_moved > 200, "{coverage:?}"); +} + +/// The public entry point (the one `process_epoch` calls) also runs its own +/// debug oracle; a fixed case drives it through both forks' drivers. +#[test] +fn drivers_run_the_oracle_on_a_crafted_epoch() { + let config = Config::mainnet(); + for (fork_number, fork) in [ForkName::Electra, ForkName::Fulu].into_iter().enumerate() { + let base = with_signing_validators_at(fork, 40); + for case in 0..20u64 { + let mut rng = SplitMix64(0xD00D << 16 | (fork_number as u64) << 8 | case); + let mut state = random_state(&mut rng, &base, &config); + // The oracle panics on divergence; errors are allowed. + let _ = + super::single_pass::process_steps_through_effective_balances(&mut state, &config); + } + } +} diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 66683060..d80b38f1 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -309,6 +309,19 @@ The leak flag is read once per step rather than once per validator, which fails some states the specification accepts; see [Spec Deviations](spec_deviations.md#the-inactivity-leak-check-runs-once-per-epoch-step-not-once-per-validator). +Electra and fulu go further (`stf::epoch::single_pass`): steps 2-9 (inactivity, +rewards, registry updates, slashings, pending deposits, effective-balance +updates) run in one loop over the registry, each validator on a local copy, with +the changes written back afterwards. What depends on more than one validator is +handled outside the loop: the exit-churn cursor is advanced locally in index +order, the pending-deposit queue is planned before the loop (top-ups of existing +validators inside it, deposits that create validators after it), and pending +consolidations run after it, with the effective-balance update of every +validator they name deferred until they are done. The genesis epoch and states +whose registry-sized lists differ in length fall back to the step-by-step path. +The debug oracle runs the specification-shaped steps on a clone and asserts the +two full states hash equal. + ## Macros and traits Two `macro_rules!` in the whole crate, both local, both replacing boilerplate that From bd4d21a8beeaece04b819aee77bb4fd243190f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:49:40 -0300 Subject: [PATCH 2/2] fix(beacon): check the effective-balance hysteresis sums balance + DOWNWARD_THRESHOLD and effective_balance + UPWARD_THRESHOLD were unchecked in both effective-balance updates, so a balance near u64::MAX panicked in debug builds and, in release, wrapped into a small sum that moved the effective balance. The specification treats that overflow as an invalid transition, and Lighthouse's single pass rejects it with safe_add. Both updates now share one checked leaves_hysteresis_band, which keeps the specification's short-circuit: the upward sum is only taken when the downward comparison fails. The fused-pass test gave nearly every state a near-u64::MAX balance, which now fails the whole epoch in both paths, so only about one state in ten carries one. Its finalized epoch also stays behind the previous epoch, as in any reachable state, now that get_finality_delay fails past it. --- .../src/beacon/stf/epoch/electra.rs | 34 +++++++-------- .../src/beacon/stf/epoch/mod.rs | 42 +++++++++++++++---- .../src/beacon/stf/epoch/single_pass.rs | 4 +- .../stf/epoch/single_pass_equivalence.rs | 16 +++++-- 4 files changed, 63 insertions(+), 33 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 2dd81711..408129d6 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -705,11 +705,8 @@ pub fn process_pending_consolidations(state: &mut BeaconState, _config: &Config) /// toward is [`get_max_effective_balance`], read per validator, rather than /// the single `MAX_EFFECTIVE_BALANCE` every validator shared before /// EIP-7251, since a compounding validator's ceiling can be far higher. The -/// hysteresis arithmetic itself is copied unchanged from that version, down -/// to computing `HYSTERESIS_INCREMENT` by dividing first and only then -/// multiplying it up to each threshold: multiplying before dividing is -/// algebraically equivalent but rounds differently, and only the -/// specification's own order reproduces its integer rounding. +/// hysteresis test itself is [`super::leaves_hysteresis_band`], shared with +/// that version. pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> { // Two passes for the same reason `super::process_effective_balance_updates` // needs them: `state` is an enum over per-fork structs, so there is no @@ -725,7 +722,7 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> .zip(state.balances().iter()) .enumerate() { - if let Some(effective) = updated_effective_balance(validator, balance) { + if let Some(effective) = updated_effective_balance(validator, balance)? { updates.push((index, effective)); } } @@ -739,20 +736,19 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> /// The effective balance `validator` moves to given its `balance`, or `None` /// when the balance is still inside the hysteresis band. -pub(super) fn updated_effective_balance(validator: &Validator, balance: Gwei) -> Option { - const HYSTERESIS_INCREMENT: Gwei = - preset::EFFECTIVE_BALANCE_INCREMENT / preset::HYSTERESIS_QUOTIENT; - const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; - const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; - - if balance + DOWNWARD_THRESHOLD < validator.effective_balance - || validator.effective_balance + UPWARD_THRESHOLD < balance - { - let max_effective_balance = get_max_effective_balance(validator); - Some((balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT).min(max_effective_balance)) - } else { - None +/// +/// Fails where [`super::leaves_hysteresis_band`] does. +pub(super) fn updated_effective_balance( + validator: &Validator, + balance: Gwei, +) -> Result> { + if !super::leaves_hysteresis_band(validator.effective_balance, balance)? { + return Ok(None); } + let max_effective_balance = get_max_effective_balance(validator); + Ok(Some( + (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT).min(max_effective_balance), + )) } // --------------------------------------------------------------------------- diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index 531c5740..1848cc50 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -31,7 +31,7 @@ mod single_pass_equivalence; use crate::beacon::containers::phase0::PendingAttestation; use crate::beacon::containers::{BeaconState, HistoricalBatch}; -use crate::beacon::error::{Result, verify}; +use crate::beacon::error::{Error, Result, verify}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ CommitteeCache, CommitteeCacheExt, get_block_root, get_block_root_at_slot, get_current_epoch, @@ -267,11 +267,6 @@ pub fn process_eth1_data_reset(state: &mut BeaconState) -> Result<()> { /// and since effective balance feeds the shuffling seed's weighting and every /// reward, that would churn far more than it measures. pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> { - const HYSTERESIS_INCREMENT: Gwei = - preset::EFFECTIVE_BALANCE_INCREMENT / preset::HYSTERESIS_QUOTIENT; - const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; - const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; - // Decided in one pass and applied in another. The state is an enum over // per-fork structs, so the accessors hand out a borrow of the whole state // rather than of one field, and there is no way to hold `validators` mutably @@ -287,9 +282,7 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> .zip(state.balances().iter()) .enumerate() { - if balance + DOWNWARD_THRESHOLD < validator.effective_balance - || validator.effective_balance + UPWARD_THRESHOLD < balance - { + if leaves_hysteresis_band(validator.effective_balance, balance)? { let effective = (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT) .min(preset::MAX_EFFECTIVE_BALANCE); updates.push((index, effective)); @@ -303,6 +296,37 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> Ok(()) } +/// Whether `balance` has moved far enough from `effective_balance` for the +/// effective balance to follow it: the hysteresis test every fork's +/// effective-balance update shares. +/// +/// `HYSTERESIS_INCREMENT` is divided first and only then multiplied up to each +/// threshold: multiplying first is algebraically equivalent but rounds +/// differently, and only the specification's order reproduces its integer +/// rounding. Both sums are checked, since the specification treats a `uint64` +/// overflow as an invalid transition, where a wrapped sum would compare as a +/// small number and move the effective balance. The upward sum is only taken +/// when the downward comparison fails, as the specification's `or` +/// short-circuits. +pub(super) fn leaves_hysteresis_band(effective_balance: Gwei, balance: Gwei) -> Result { + const HYSTERESIS_INCREMENT: Gwei = + preset::EFFECTIVE_BALANCE_INCREMENT / preset::HYSTERESIS_QUOTIENT; + const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; + const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; + + let below = balance + .checked_add(DOWNWARD_THRESHOLD) + .ok_or(Error::ArithmeticOverflow("balance + DOWNWARD_THRESHOLD"))? + < effective_balance; + Ok(below + || effective_balance + .checked_add(UPWARD_THRESHOLD) + .ok_or(Error::ArithmeticOverflow( + "effective_balance + UPWARD_THRESHOLD", + ))? + < balance) +} + /// Zeroes the slot the slashings ring buffer is about to reuse. pub fn process_slashings_reset(state: &mut BeaconState) -> Result<()> { let next_epoch = get_current_epoch(state) + 1; diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs index ef927771..f83bc79f 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass.rs @@ -239,7 +239,7 @@ fn fused(state: &mut BeaconState, config: &Config) -> Result<()> { while deferred_cursor.next_if(|&&at| at < index).is_some() {} let is_deferred = deferred_cursor.next_if(|&&at| at == index).is_some(); if !is_deferred - && let Some(effective) = updated_effective_balance(¤t, new_balance) + && let Some(effective) = updated_effective_balance(¤t, new_balance)? && effective != current.effective_balance { current.to_mut().effective_balance = effective; @@ -287,7 +287,7 @@ fn fused(state: &mut BeaconState, config: &Config) -> Result<()> { for index in patched { let validator = state.validator(index as ValidatorIndex)?; let balance = state.balance(index as ValidatorIndex)?; - if let Some(effective) = updated_effective_balance(validator, balance) { + if let Some(effective) = updated_effective_balance(validator, balance)? { effective_updates.push((index, effective)); } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs index c2111915..205f4577 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/single_pass_equivalence.rs @@ -23,6 +23,7 @@ use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; use crate::beacon::containers::shared::{Checkpoint, DepositMessage}; use crate::beacon::containers::{BeaconState, electra as electra_containers}; use crate::beacon::fork::ForkName; +use crate::beacon::helpers::accessors::get_previous_epoch; use crate::beacon::helpers::electra::ExitChurnCursor; use crate::beacon::helpers::misc::{ compute_deposit_domain, compute_signing_root, compute_start_slot_at_epoch, @@ -129,19 +130,28 @@ fn random_state(rng: &mut SplitMix64, base: &BeaconState, config: &Config) -> Be let hysteresis_up = INCREMENT / preset::HYSTERESIS_QUOTIENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; let participation_density = rng.pick(&[0, 30, 70, 95, 100]); + // A balance within a hysteresis threshold of `u64::MAX` overflows the + // effective-balance update and fails the whole epoch, so only a few states + // carry one: enough to check both paths fail together, few enough to + // leave the rest to compare. + let extreme_balances = rng.chance(10); let near = |rng: &mut SplitMix64| (epoch + rng.below(5)).saturating_sub(2); - // Finality first: the activation branch reads the finalized epoch. + // Finality first: the activation branch reads the finalized epoch. It + // stays at or behind the previous epoch, as in any reachable state: + // `get_finality_delay` subtracts it from the previous epoch, and fails + // otherwise. let checkpoint = |epoch: u64| Checkpoint { epoch, root: Root::ZERO, }; + let previous_epoch = get_previous_epoch(&state); *state.previous_justified_checkpoint_mut() = checkpoint(epoch.saturating_sub(rng.below(4))); *state.current_justified_checkpoint_mut() = checkpoint(epoch.saturating_sub(rng.below(3))); let finalized_epoch = if rng.chance(30) { 0 } else { - epoch.saturating_sub(rng.below(3)) + previous_epoch.saturating_sub(rng.below(3)) }; *state.finalized_checkpoint_mut() = checkpoint(finalized_epoch); for bit in 0..constants::JUSTIFICATION_BITS_LENGTH { @@ -201,7 +211,7 @@ fn random_state(rng: &mut SplitMix64, base: &BeaconState, config: &Config) -> Be 4 => effective + hysteresis_up + 1, 5 => effective + hysteresis_up, 6 => effective + rng.below(3 * INCREMENT), - 7 if rng.chance(20) => u64::MAX - rng.below(3), + 7 if extreme_balances && rng.chance(20) => u64::MAX - rng.below(3), _ => effective, }); }