diff --git a/crates/blockchain/src/epoch_precompute.rs b/crates/blockchain/src/epoch_precompute.rs new file mode 100644 index 00000000..f5ffbb6a --- /dev/null +++ b/crates/blockchain/src/epoch_precompute.rs @@ -0,0 +1,343 @@ +//! Beacon epoch-transition precompute. +//! +//! The first block of an epoch runs `process_epoch` inline on the import path, +//! then pays the post-epoch rehash for its state-root check. Both can be done +//! before the block arrives, since the parent is known by the end of the last +//! slot of the previous epoch: clone the head state, advance it to the epoch's +//! first slot on a blocking worker, hash it, and park it in the store's state +//! cache under [`CacheKey::CheckpointState`]. `fork_choice::on_block` resumes +//! from that entry when it finds one, and falls back to the inline path when it +//! does not, so the precompute is a pure speed trade. +//! +//! Two triggers feed the same worker: +//! +//! | Trigger | When | Covers | +//! |---|---|---| +//! | [`Trigger::Head`] | an import leaves a last-slot block as head | a late last-slot block | +//! | [`Trigger::Timer`] | three quarters into a last slot | a skipped last slot | +//! +//! At most one worker runs at a time, and a key already in the cache is never +//! recomputed. A block that arrives while the worker is still running imports +//! inline; the late result is stored and simply goes unused by that block. + +use std::sync::Arc; +use std::time::Duration; + +use ethlambda_state_transition::beacon::fork_choice; +use ethlambda_state_transition::beacon::helpers::misc::compute_start_slot_at_epoch; +use ethlambda_state_transition::metrics as stf_metrics; +use ethlambda_storage::{CacheKey, Chain}; +use ethlambda_types::ShortRoot; +use ethlambda_types::beacon::containers::BeaconState; +use ethlambda_types::beacon::preset; +use ethlambda_types::primitives::H256; +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{Context, Handler, send_after}; +use tracing::{debug, info, warn}; + +use crate::BlockChainServer; + +/// What caused a precompute, and the label it is counted under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Trigger { + /// An import made a last-slot block the head. + Head, + /// The three-quarter point of a last slot, carrying that slot. + Timer { slot: u64 }, +} + +impl Trigger { + fn label(self) -> &'static str { + match self { + Trigger::Head => "head", + Trigger::Timer { .. } => "timer", + } + } +} + +/// The state a precompute produces: the head's post-state advanced to the first +/// slot of `epoch`. The same identity as `CacheKey::CheckpointState`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct PrecomputeKey { + pub(crate) epoch: u64, + pub(crate) root: H256, +} + +/// Everything [`decide`] reads, gathered so the decision is a pure function. +pub(crate) struct DecisionInputs { + pub(crate) trigger: Trigger, + pub(crate) syncing: bool, + pub(crate) head_slot: u64, + pub(crate) head_root: H256, + /// The wall-clock slot now. + pub(crate) current_slot: u64, + /// The worker running right now, if any. + pub(crate) in_flight: Option, + /// Whether the store already holds a state for the candidate key. + pub(crate) already_cached: bool, +} + +fn is_last_slot_of_epoch(slot: u64) -> bool { + (slot + 1).is_multiple_of(preset::SLOTS_PER_EPOCH) +} + +/// Whether to start a precompute, and for which key. +/// +/// `None` when the node is syncing, when the trigger does not sit on the last +/// slot of an epoch, when a worker is already running (at most one), or when +/// the store already holds the state. A head trigger also requires the head to +/// be fresh (at most one slot behind the wall clock): the sync tracker moves +/// once per slot, so a follower importing history would otherwise precompute +/// every epoch it replays. +pub(crate) fn decide(inputs: &DecisionInputs) -> Option { + if inputs.syncing || inputs.in_flight.is_some() { + return None; + } + let epoch = match inputs.trigger { + Trigger::Head => { + if !is_last_slot_of_epoch(inputs.head_slot) + || inputs.current_slot > inputs.head_slot.saturating_add(1) + { + return None; + } + inputs.head_slot / preset::SLOTS_PER_EPOCH + 1 + } + Trigger::Timer { slot } => { + // A head past the slot would be a stale message racing a newer + // import; one already at the boundary has nothing left to advance. + if !is_last_slot_of_epoch(slot) || inputs.head_slot > slot { + return None; + } + slot / preset::SLOTS_PER_EPOCH + 1 + } + }; + if inputs.head_slot >= compute_start_slot_at_epoch(epoch) { + return None; + } + let key = PrecomputeKey { + epoch, + root: inputs.head_root, + }; + (!inputs.already_cached).then_some(key) +} + +/// Delay from the start of a slot to its three-quarter point. +pub(crate) fn three_quarter_slot(slot_duration_ms: u64) -> Duration { + Duration::from_millis(slot_duration_ms / 4 * 3) +} + +/// Self-message armed at a last slot's start, delivered at its three-quarter point. +pub(crate) struct EpochPrecomputeCheck { + pub(crate) slot: u64, +} +impl Message for EpochPrecomputeCheck { + type Result = (); +} + +/// A worker's result, sent back to the actor. +pub(crate) struct EpochPrecomputed { + pub(crate) key: PrecomputeKey, + pub(crate) result: Result, String>, +} +impl Message for EpochPrecomputed { + type Result = (); +} + +impl BlockChainServer { + /// Arm the three-quarter check if `slot` is the last of its epoch. + /// + /// Called from the once-per-slot tick, which fires at the slot boundary, so + /// the delay is the three-quarter point less whatever of the slot the tick + /// has already consumed (a tick delayed by a long import). + pub(crate) fn arm_epoch_precompute_check(&self, slot: u64, ctx: &Context) { + if self.store.chain() != Chain::Beacon || !is_last_slot_of_epoch(slot) { + return; + } + let slot_duration_ms = self.store.config().time_grid().milliseconds_per_slot; + let into_slot = Duration::from_millis(self.ms_into_slot(slot).max(0) as u64); + let delay = three_quarter_slot(slot_duration_ms).saturating_sub(into_slot); + send_after(delay, ctx.clone(), EpochPrecomputeCheck { slot }); + } + + /// Start a precompute for the current head if [`decide`] allows one. + pub(crate) fn maybe_start_epoch_precompute(&mut self, trigger: Trigger, ctx: &Context) { + if self.store.chain() != Chain::Beacon { + return; + } + let Some((head_slot, head_root)) = self.store.beacon_head() else { + return; + }; + let candidate_epoch = match trigger { + Trigger::Head => head_slot / preset::SLOTS_PER_EPOCH + 1, + Trigger::Timer { slot } => slot / preset::SLOTS_PER_EPOCH + 1, + }; + let already_cached = self + .store + .cached_state(CacheKey::CheckpointState { + epoch: candidate_epoch, + root: head_root, + }) + .is_some(); + let inputs = DecisionInputs { + trigger, + syncing: self.sync_status.is_syncing(), + head_slot, + head_root, + current_slot: self.wall_clock_slot(), + in_flight: self.epoch_precompute_in_flight, + already_cached, + }; + let Some(key) = decide(&inputs) else { + return; + }; + // The head's post-state is resident (it was just imported); a read that + // cannot find it means the head moved to a block this node has no state + // for, which a later trigger can retry. + let Ok(Some(head_state)) = self.store.get_state(&head_root) else { + return; + }; + + self.epoch_precompute_in_flight = Some(key); + stf_metrics::inc_epoch_precompute_started(trigger.label()); + debug!( + epoch = key.epoch, + head_slot, + head_root = %ShortRoot(&head_root.0), + trigger = trigger.label(), + "Starting epoch precompute" + ); + + let config = self.store.config(); + let actor = ctx.actor_ref(); + tokio::task::spawn_blocking(move || { + let result = { + let _timing = stf_metrics::time_epoch_precompute(); + fork_choice::advance_to_epoch_start(&head_state, key.epoch, &config) + }; + let result = result.map(Arc::new).map_err(|err| err.to_string()); + let _ = actor.send(EpochPrecomputed { key, result }); + }); + } +} + +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: EpochPrecomputeCheck, ctx: &Context) { + self.maybe_start_epoch_precompute(Trigger::Timer { slot: msg.slot }, ctx); + } +} + +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: EpochPrecomputed, _ctx: &Context) { + self.epoch_precompute_in_flight = None; + match msg.result { + Ok(state) => { + // Stored even if the block it was meant for already imported + // inline: attestation targets for the epoch read the same key. + self.store.cache_state( + CacheKey::CheckpointState { + epoch: msg.key.epoch, + root: msg.key.root, + }, + state, + ); + info!( + epoch = msg.key.epoch, + head_root = %ShortRoot(&msg.key.root.0), + "Epoch precompute stored" + ); + } + Err(err) => warn!( + epoch = msg.key.epoch, + head_root = %ShortRoot(&msg.key.root.0), + %err, + "Epoch precompute failed" + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPE: u64 = preset::SLOTS_PER_EPOCH; + + fn inputs(trigger: Trigger, head_slot: u64) -> DecisionInputs { + DecisionInputs { + trigger, + syncing: false, + head_slot, + head_root: H256([7; 32]), + current_slot: head_slot, + in_flight: None, + already_cached: false, + } + } + + #[test] + fn a_last_slot_head_precomputes_the_next_epoch() { + let key = decide(&inputs(Trigger::Head, 2 * SPE - 1)).expect("starts"); + assert_eq!(key.epoch, 2); + assert_eq!(key.root, H256([7; 32])); + } + + #[test] + fn a_head_elsewhere_in_the_epoch_starts_nothing() { + assert!(decide(&inputs(Trigger::Head, SPE)).is_none()); + assert!(decide(&inputs(Trigger::Head, 2 * SPE - 2)).is_none()); + } + + #[test] + fn a_syncing_node_starts_nothing() { + let mut i = inputs(Trigger::Head, 2 * SPE - 1); + i.syncing = true; + assert!(decide(&i).is_none()); + } + + #[test] + fn a_stale_head_trigger_starts_nothing() { + let mut i = inputs(Trigger::Head, 2 * SPE - 1); + i.current_slot = 2 * SPE + 5; + assert!(decide(&i).is_none()); + } + + #[test] + fn a_running_worker_blocks_a_second_one() { + let mut i = inputs(Trigger::Head, 2 * SPE - 1); + i.in_flight = Some(PrecomputeKey { + epoch: 2, + root: H256([7; 32]), + }); + assert!(decide(&i).is_none()); + } + + #[test] + fn a_state_already_cached_is_not_recomputed() { + let mut i = inputs(Trigger::Timer { slot: 2 * SPE - 1 }, 2 * SPE - 3); + i.already_cached = true; + assert!(decide(&i).is_none()); + } + + #[test] + fn the_timer_covers_a_skipped_last_slot() { + let key = + decide(&inputs(Trigger::Timer { slot: 2 * SPE - 1 }, 2 * SPE - 3)).expect("starts"); + assert_eq!(key.epoch, 2); + } + + #[test] + fn the_timer_ignores_slots_that_are_not_the_last() { + assert!(decide(&inputs(Trigger::Timer { slot: SPE }, SPE - 1)).is_none()); + } + + #[test] + fn the_timer_skips_a_head_already_at_the_boundary() { + assert!(decide(&inputs(Trigger::Timer { slot: 2 * SPE - 1 }, 2 * SPE)).is_none()); + } + + #[test] + fn three_quarters_follows_the_configured_slot_duration() { + assert_eq!(three_quarter_slot(12_000), Duration::from_secs(9)); + assert_eq!(three_quarter_slot(6_000), Duration::from_millis(4_500)); + } +} diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 969420ae..534fd9be 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -62,6 +62,7 @@ mod beacon_payloads; pub use beacon_payloads::checkpoint_hash; pub mod block_builder; pub(crate) mod coverage; +mod epoch_precompute; pub mod events; pub(crate) mod fork_choice_tree; pub mod import_timing; @@ -420,6 +421,7 @@ impl BlockChain { engine, safe_slots_to_import_optimistically, last_tick_instant: None, + epoch_precompute_in_flight: None, sync_status, sync_status_controller, events, @@ -546,6 +548,10 @@ pub struct BlockChainServer { /// Last tick instant for measuring interval duration. last_tick_instant: Option, + /// The epoch precompute worker running right now, if any (beacon only). At + /// most one runs at a time; see [`epoch_precompute`]. + epoch_precompute_in_flight: Option, + /// Stateful sync heuristic used by `lean_node_sync_status`. Also gates /// validator duties while syncing, unless that gating was disabled at /// startup via `--disable-duty-sync-gate` (then it is metric-only). On a @@ -1059,6 +1065,12 @@ impl BlockChainServer { self.redrive_missing_envelopes(); self.settle_envelopes().await; + // Beacon only (both calls are no-ops on lean): a last slot arms its + // three-quarter precompute check, and a redrive above may have made a + // last-slot block the head without passing through the block handler. + self.arm_epoch_precompute_check(slot, ctx); + self.maybe_start_epoch_precompute(epoch_precompute::Trigger::Head, ctx); + // Per-interval duties for this tick. Lean-only, so this is where a // beacon follower's tick ends: it has no validator duties (see // [`ChainDuties::Beacon`]), and everything a tick owes it happened @@ -1983,6 +1995,7 @@ impl BlockChainServer { engine, safe_slots_to_import_optimistically, last_tick_instant: None, + epoch_precompute_in_flight: None, sync_status: SyncStatusTracker::new(false), sync_status_controller: SyncStatusController::default(), events: EventBus::default(), @@ -3829,6 +3842,10 @@ impl Handler for BlockChainServer { self.on_block(msg.block, timings).await; // A gloas envelope that arrived ahead of this block is judgeable now. self.settle_envelopes().await; + + // The import above may have left a last-slot block as head: the + // moment to start the next epoch's transition (no-op on lean). + self.maybe_start_epoch_precompute(epoch_precompute::Trigger::Head, ctx); } } @@ -4372,6 +4389,7 @@ mod tests { engine: None, safe_slots_to_import_optimistically: constants::SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, last_tick_instant: None, + epoch_precompute_in_flight: None, sync_status: SyncStatusTracker::new(false), sync_status_controller: SyncStatusController::default(), events: EventBus::default(), diff --git a/crates/blockchain/src/sync_status.rs b/crates/blockchain/src/sync_status.rs index 5e968a5c..27dff999 100644 --- a/crates/blockchain/src/sync_status.rs +++ b/crates/blockchain/src/sync_status.rs @@ -122,6 +122,12 @@ impl SyncStatusTracker { } } + /// Whether the tracker currently considers the node to be syncing, + /// regardless of whether that gates duties. + pub(crate) fn is_syncing(&self) -> bool { + self.syncing + } + pub(crate) fn duties_allowed(&self) -> bool { // Gate disabled: the syncing state is observe-only, never suppresses duties. !self.gate_duties || !self.syncing diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 59718c41..3bc4a41e 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -196,6 +196,7 @@ use crate::beacon::primitives::{ ValidatorIndex, }; use crate::beacon::stf; +use crate::metrics; // --------------------------------------------------------------------------- // LatestMessage, PowBlock, PayloadStatusV1, ForkChoiceNode, PayloadStatus @@ -210,8 +211,8 @@ use crate::beacon::stf; // keyed on plain `Root`s, and nothing about either type needs storage of its // own beyond what `BeaconScratch` already keeps. pub use ethlambda_types::beacon::fork_choice::{ - BlockPayloadLink, ForkChoiceNode, LatestMessage, PayloadStatus, PayloadStatusEnum, - PayloadStatusV1, PowBlock, + BlockPayloadLink, ForkChoiceNode, JustifiedBalances, LatestMessage, PayloadStatus, + PayloadStatusEnum, PayloadStatusV1, PowBlock, }; // --------------------------------------------------------------------------- @@ -996,6 +997,156 @@ pub fn checkpoint_state( Ok(state) } +/// The state of `parent_root` advanced to the first slot of `epoch`, computed +/// without the store. +/// +/// What [`checkpoint_state`] derives on a miss, taking the checkpoint block's +/// post-state as an argument so a worker thread can run it off the actor. +/// Hashes the result after flushing buffered writes, so the import that later +/// resumes from a clone of it finds the tree nodes' hashes already memoized +/// and pays only for the flat fields. A state already at or past the epoch's +/// first slot comes back unchanged, as `checkpoint_state` returns it. +pub fn advance_to_epoch_start( + parent_state: &BeaconState, + epoch: Epoch, + config: &Config, +) -> Result { + let target_slot = compute_start_slot_at_epoch(epoch); + let mut state = parent_state.clone(); + if state.slot() < target_slot { + stf::process_slots(&mut state, target_slot, config)?; + } + state.apply_pending_mutations(); + let _ = state.hash_tree_root(); + Ok(state) +} +/// Applies `signed_block` to a copy of its parent's post-state, resuming from +/// the precomputed epoch-boundary state when the block crosses into an epoch +/// and one is cached. +/// +/// Returns the copy with the transition's outcome, since a failed transition +/// still leaves the caller a state to inspect. Both starting points reach the +/// same post-state: the precomputed one is what `process_slots` would produce +/// from the parent. Either way the copy is independent, so a failing block +/// cannot corrupt a cached entry. +/// +/// This is also where the epoch-boundary balances of a crossing block are +/// built (see [`cache_boundary_balances`]): a block with empty slots between +/// its parent and its epoch's first slot makes `(epoch, parent_root)` an +/// epoch-boundary checkpoint, and the boundary state is right here. A +/// precompute hit builds from the precomputed state; a miss advances the +/// parent's copy to the epoch start once, builds from that, and carries on +/// from there to the block's slot, so `process_slots` still runs each slot +/// once. The balances come before the block is validated, which is harmless: +/// the boundary state is a function of the parent alone, so an invalid block +/// cannot make the entry wrong, and the key names a real parent either way. +#[allow(clippy::too_many_arguments)] +fn transition_block( + store: &Store, + parent_state: &BeaconState, + parent_root: Root, + signed_block: &SignedBeaconBlock, + validate_result: bool, + config: &Config, + engine: &stf::ExecutionEngine, + committees: &CommitteeCache, +) -> (BeaconState, Result<()>) { + let block_slot = signed_block.slot(); + let epoch = compute_epoch_at_slot(block_slot); + let epoch_start = compute_start_slot_at_epoch(epoch); + // The epoch's first slot (or more) was empty: the parent is the epoch's + // checkpoint block. A block *at* the first slot is its own, cached from + // its post-state by the caller. + let crosses_with_empty_slots = parent_state.slot() < epoch_start && epoch_start < block_slot; + let boundary = Checkpoint { + epoch, + root: parent_root, + }; + + let Some(precomputed) = + precomputed_epoch_state(store, parent_state.slot(), parent_root, block_slot) + else { + let mut state = parent_state.clone(); + if !crosses_with_empty_slots { + let outcome = stf::state_transition( + &mut state, + signed_block, + validate_result, + config, + engine, + committees, + ); + return (state, outcome); + } + // The same `process_slots` steps `state_transition` would run, split + // at the epoch start so the boundary state can be read in between. + let outcome = stf::process_slots(&mut state, epoch_start, config) + .inspect(|()| cache_boundary_balances(store, boundary, &state)) + .and_then(|()| stf::process_slots(&mut state, block_slot, config)) + .and_then(|()| { + stf::apply_block( + &mut state, + signed_block, + validate_result, + config, + engine, + committees, + ) + }); + return (state, outcome); + }; + + if crosses_with_empty_slots { + cache_boundary_balances(store, boundary, &precomputed); + } + let mut state = (*precomputed).clone(); + // Only the slots skipped past the boundary are left to advance; none when + // the block sits on the boundary itself. + let outcome = if state.slot() < block_slot { + stf::process_slots(&mut state, block_slot, config) + } else { + Ok(()) + } + .and_then(|()| { + stf::apply_block( + &mut state, + signed_block, + validate_result, + config, + engine, + committees, + ) + }); + (state, outcome) +} + +/// The precomputed epoch-boundary state an import can resume from, if any. +/// +/// Applies to a block that crosses into a new epoch: its parent's post-state +/// is before the first slot of the block's epoch, and the block is at or after +/// that slot. The precompute worker stores it under the key +/// [`checkpoint_state`] uses for `(block_epoch, parent_root)`, since the two +/// name the same state. Counts a hit or a miss for every such import and for +/// no other. +fn precomputed_epoch_state( + store: &Store, + parent_slot: Slot, + parent_root: Root, + block_slot: Slot, +) -> Option> { + let epoch = compute_epoch_at_slot(block_slot); + let epoch_start = compute_start_slot_at_epoch(epoch); + if !(parent_slot < epoch_start && epoch_start <= block_slot) { + return None; + } + let cached = store.cached_state(CacheKey::CheckpointState { + epoch, + root: parent_root, + }); + crate::metrics::inc_epoch_precompute_lookups(if cached.is_some() { "hit" } else { "miss" }); + cached +} + // --------------------------------------------------------------------------- // get_forkchoice_store // --------------------------------------------------------------------------- @@ -1225,8 +1376,18 @@ pub fn get_ancestor(index: &HashMap, root: Root, slot: Slot) /// values it is called with are themselves already expressed on a 0-100 /// scale. pub fn calculate_committee_fraction(state: &BeaconState, committee_percent: u64) -> Result { - let committee_weight = get_total_active_balance(state)? / preset::SLOTS_PER_EPOCH; - Ok(committee_weight.saturating_mul(committee_percent) / 100) + Ok(committee_fraction( + get_total_active_balance(state)?, + committee_percent, + )) +} + +/// The arithmetic of [`calculate_committee_fraction`], for a caller that +/// already holds the total active balance (the [`JustifiedBalances`] snapshot's +/// own). +pub fn committee_fraction(total_active_balance: Gwei, committee_percent: u64) -> Gwei { + let committee_weight = total_active_balance / preset::SLOTS_PER_EPOCH; + committee_weight.saturating_mul(committee_percent) / 100 } /// The checkpoint block for `epoch`, on `root`'s chain: the ancestor of `root` @@ -1247,10 +1408,113 @@ pub fn get_checkpoint_block( /// See [`calculate_committee_fraction`] for why this divides by a bare /// `100` rather than [`constants::BASIS_POINTS`]. pub fn get_proposer_score(store: &Store, config: &Config) -> Result { - let justified_checkpoint = store.beacon_justified_checkpoint(); - let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; - let committee_weight = get_total_active_balance(&justified_state)? / preset::SLOTS_PER_EPOCH; - Ok(committee_weight.saturating_mul(config.proposer_score_boost) / 100) + let balances = justified_balances(store, config)?; + Ok(committee_fraction( + balances.total_active_balance(), + config.proposer_score_boost, + )) +} + +/// The flat balances of the store's justified checkpoint state, looked up in +/// the store's per-checkpoint cache. +/// +/// Block import normally filled it already ([`cache_boundary_balances`] and +/// [`cache_first_slot_balances`]), so the tick that moves the justified +/// checkpoint costs a lookup and a head computation reads no state. A miss, +/// after a restart or for a checkpoint whose boundary import this node never +/// saw, builds it from [`checkpoint_state`] and caches the result. +/// +/// Keyed by the checkpoint itself, which every writer of the justified +/// checkpoint (`update_checkpoints` from three handlers, store construction, +/// restart) already changes, so none of them needs a hook. Both sources build +/// from the state [`checkpoint_state`] returns, the one `get_weight` reads, so +/// the snapshot is exactly what the specification's per-root definition sees; +/// a later post-state of the same epoch would not be, since `slashed` can +/// differ. A failed `checkpoint_state` fails the call, as it does for every +/// other fork-choice reader, and the head stays where it is. +pub fn justified_balances(store: &Store, config: &Config) -> Result> { + let checkpoint = store.beacon_justified_checkpoint(); + if let Some(balances) = store.justified_balances(&checkpoint) { + metrics::inc_justified_balances_lookups("hit"); + return Ok(balances); + } + metrics::inc_justified_balances_lookups("miss"); + let state = checkpoint_state(store, &checkpoint, config)?; + let balances = Arc::new(build_justified_balances(checkpoint, &state)); + store.insert_justified_balances(Arc::clone(&balances)); + Ok(balances) +} + +/// Caches the balances of the epoch-boundary checkpoint `checkpoint` from +/// `state`, the checkpoint's state at the epoch's first slot, unless the cache +/// has them. +fn cache_boundary_balances(store: &Store, checkpoint: Checkpoint, state: &BeaconState) { + if store.justified_balances(&checkpoint).is_none() { + let balances = Arc::new(build_justified_balances(checkpoint, state)); + store.insert_justified_balances(balances); + } +} + +/// A block at its epoch's first slot is its own epoch-boundary block, and its +/// post-state is what [`checkpoint_state`] returns for `(epoch, block_root)` +/// (no advance needed): cache the balances from it. +fn cache_first_slot_balances(store: &Store, post_state: &BeaconState, block_root: Root) { + let slot = post_state.slot(); + let epoch = compute_epoch_at_slot(slot); + if slot != compute_start_slot_at_epoch(epoch) { + return; + } + let checkpoint = Checkpoint { + epoch, + root: block_root, + }; + cache_boundary_balances(store, checkpoint, post_state); +} + +/// One in-order pass over `state`'s registry: each validator's vote weight +/// (zero if inactive or slashed), its raw effective balance in increments, and, +/// in the same pass, the total active balance with exactly the semantics of +/// `get_total_active_balance` (slashed-but-active validators count, +/// saturating, floored at one increment). +fn build_justified_balances(checkpoint: Checkpoint, state: &BeaconState) -> JustifiedBalances { + let _timing = metrics::time_justified_balances_build(); + // Activity at the state's own epoch, as `get_weight` reads it; not asserted + // equal to `checkpoint.epoch`, which the unit-test stores do not keep. + let epoch = get_current_epoch(state); + let mut total: Gwei = 0; + let mut increments: Vec = Vec::new(); + let balances = state + .iter_validators() + .map(|validator| { + // An effective balance is a whole number of increments no larger + // than `MAX_EFFECTIVE_BALANCE_ELECTRA`; the snapshot's exactness + // depends on it. + assert_eq!( + validator.effective_balance % preset::EFFECTIVE_BALANCE_INCREMENT, + 0, + "effective balance is a whole number of increments" + ); + increments.push( + u16::try_from(validator.effective_balance / preset::EFFECTIVE_BALANCE_INCREMENT) + .expect("effective balance fits u16 increments"), + ); + if !is_active_validator(validator, epoch) { + return 0; + } + total = total.saturating_add(validator.effective_balance); + if validator.slashed { + 0 + } else { + validator.effective_balance + } + }) + .collect(); + JustifiedBalances::new( + checkpoint, + balances, + total.max(preset::EFFECTIVE_BALANCE_INCREMENT), + increments.into(), + ) } /// The effective balance of every non-equivocating, active, unslashed @@ -1310,6 +1574,38 @@ pub fn get_attestation_score( Ok(attestation_score) } +/// [`get_attestation_score`] read from the justified-balances snapshot instead +/// of the justified state: the same sum, since the snapshot holds a zero for +/// exactly the validators the state version skips (inactive, slashed), and +/// equivocators are dropped by the store. Reads no state, so the head path and +/// the reorg checks never need the justified checkpoint's. +fn snapshot_attestation_score( + store: &Store, + index: &HashMap, + root: Root, + balances: &JustifiedBalances, +) -> Result { + let block_slot = index + .get(&root) + .ok_or(Error::SpecAssert("root in store.blocks"))? + .0; + let mut attestation_score: Gwei = 0; + // Nothing that reads the store's own scratch may run inside the closure: + // see `for_each_non_equivocating_latest_message`. `get_ancestor` reads only + // `index`. + store.for_each_non_equivocating_latest_message(|validator_index, message| { + let balance = balances.get(validator_index); + if balance == 0 { + return; + } + if matches!(get_ancestor(index, message.root, block_slot), Ok(ancestor) if ancestor == root) + { + attestation_score = attestation_score.saturating_add(balance); + } + }); + Ok(attestation_score) +} + /// The LMD GHOST weight of `root`: [`get_attestation_score`] against the /// justified checkpoint's state, plus the proposer boost if it applies. /// @@ -1378,8 +1674,7 @@ pub fn compute_weights( config: &Config, ) -> Result> { let justified_checkpoint = store.beacon_justified_checkpoint(); - let state = checkpoint_state(store, &justified_checkpoint, config)?; - let current_epoch = get_current_epoch(&state); + let balances = justified_balances(store, config)?; // Keyed on the voted block itself; the fold below turns these into subtree // totals in place. @@ -1388,19 +1683,15 @@ pub fn compute_weights( // `for_each_non_equivocating_latest_message` for why asking it per voter // from in here would deadlock. store.for_each_non_equivocating_latest_message(|validator_index, message| { - // Not `get_active_validator_indices`: that allocates the whole active - // set (~2 million entries on mainnet) to answer a membership question, - // and an index past this state's registry is a validator that did not - // exist yet at the justified checkpoint, which is a skip rather than an - // error. - let Ok(validator) = state.validator(validator_index) else { - return; - }; - if validator.slashed || !is_active_validator(validator, current_epoch) { + // An index past the snapshot is a validator that did not exist at the + // justified checkpoint, and an inactive or slashed one is zero in it: + // both are a skip rather than an error. + let balance = balances.get(validator_index); + if balance == 0 { return; } let entry = weights.entry(message.root).or_default(); - *entry = entry.saturating_add(validator.effective_balance); + *entry = entry.saturating_add(balance); }); // Highest slot first: see above for why that is a topological order. @@ -1432,7 +1723,8 @@ pub fn compute_weights( let justified_slot = index .get(&justified_checkpoint.root) .map_or(0, |(slot, _)| *slot); - let proposer_score = get_proposer_score(store, config)?; + let proposer_score = + committee_fraction(balances.total_active_balance(), config.proposer_score_boost); let mut cursor = boost_root; while let Some((slot, parent_root)) = index.get(&cursor).copied() { let entry = weights.entry(cursor).or_default(); @@ -1854,9 +2146,11 @@ pub fn compute_node_weights( rules: ForkRules, ) -> Result { let tree = PayloadTree { store, rules }; - let justified_checkpoint = store.beacon_justified_checkpoint(); - let state = checkpoint_state(store, &justified_checkpoint, config)?; - let current_epoch = get_current_epoch(&state); + // The vote loop, the boost score and the gloas boost gate all read the + // justified checkpoint's snapshot, so a head walk reads no justified state: + // on a hit (the usual case, since import builds it) not even a lookup of + // one. The gate's committees come from the head and parent *block* states. + let balances = justified_balances(store, config)?; let bound = walk_bound(store, index); let in_window = |root: &Root| index.get(root).is_some_and(|&(slot, _)| slot >= bound); @@ -1866,19 +2160,18 @@ pub fn compute_node_weights( // from in here would deadlock, and a payload link is such a read. let mut votes: HashMap<(Root, Slot, bool), Gwei> = HashMap::new(); store.for_each_non_equivocating_latest_message(|validator_index, message| { - // Not `get_active_validator_indices`, for the reason - // `compute_weights` gives; a validator past this state's registry - // did not exist at the justified checkpoint, which is a skip. - let Ok(validator) = state.validator(validator_index) else { - return; - }; - if validator.slashed || !is_active_validator(validator, current_epoch) { + // An index past the snapshot is a validator that did not exist at the + // justified checkpoint, and an inactive or slashed one is zero in it: + // both are a skip rather than an error, as is a zero balance, which + // adds nothing either way. + let balance = balances.get(validator_index); + if balance == 0 { return; } let entry = votes .entry((message.root, message.slot, message.payload_present)) .or_default(); - *entry = entry.saturating_add(validator.effective_balance); + *entry = entry.saturating_add(balance); }); let mut weights = NodeWeights { @@ -1960,14 +2253,19 @@ pub fn compute_node_weights( let boost_root = store.proposer_boost_root(); let boost_applies = match rules { ForkRules::PreGloas => !boost_root.is_zero() && in_window(&boost_root), - ForkRules::Gloas => { - should_apply_proposer_boost_with(store, config, committees, index, &state, |parent| { + ForkRules::Gloas => should_apply_proposer_boost_with( + store, + config, + committees, + index, + &balances, + |parent| { Ok(weights.raw(ForkChoiceNode { root: parent, payload_status: PayloadStatus::Pending, })) - })? - } + }, + )?, }; if boost_applies { let proposer_score = get_proposer_score(store, config)?; @@ -2356,10 +2654,9 @@ pub fn is_head_weak( config: &Config, committees: &CommitteeCache, ) -> Result { - let justified_checkpoint = store.beacon_justified_checkpoint(); - let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; + let balances = justified_balances(store, config)?; let index = store.block_index(); - let attestation_score = get_attestation_score(store, &index, head_root, &justified_state)?; + let attestation_score = snapshot_attestation_score(store, &index, head_root, &balances)?; let &(head_slot, _) = index .get(&head_root) .ok_or(Error::SpecAssert("head_root in store.blocks"))?; @@ -2369,7 +2666,7 @@ pub fn is_head_weak( head_slot, config, committees, - &justified_state, + &balances, attestation_score, ) } @@ -2390,11 +2687,13 @@ fn is_head_weak_with( head_slot: Slot, config: &Config, committees: &CommitteeCache, - justified_state: &BeaconState, + balances: &JustifiedBalances, attestation_score: Gwei, ) -> Result { - let reorg_threshold = - calculate_committee_fraction(justified_state, config.reorg_head_weight_threshold)?; + let reorg_threshold = committee_fraction( + balances.total_active_balance(), + config.reorg_head_weight_threshold, + ); let mut head_weight = attestation_score; let head_state = store @@ -2406,8 +2705,11 @@ fn is_head_weak_with( for committee_index in 0..epoch_committees.committees_per_slot() { for &validator_index in epoch_committees.committee(head_slot, committee_index)? { if store.is_equivocating(validator_index) { - let validator = justified_state.validator(validator_index)?; - head_weight = head_weight.saturating_add(validator.effective_balance); + // The raw effective balance at the justified state: the + // snapshot's vote weight is zero for a slashed validator, and + // an equivocator often is. + head_weight = + head_weight.saturating_add(balances.effective_balance(validator_index)); } } } @@ -2424,15 +2726,16 @@ fn is_head_weak_with( /// its own purposes and this function's own lookup cannot disagree about /// which block that is. pub fn is_parent_strong(store: &Store, root: Root, config: &Config) -> Result { - let justified_checkpoint = store.beacon_justified_checkpoint(); - let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; - let parent_threshold = - calculate_committee_fraction(&justified_state, config.reorg_parent_weight_threshold)?; + let balances = justified_balances(store, config)?; + let parent_threshold = committee_fraction( + balances.total_active_balance(), + config.reorg_parent_weight_threshold, + ); let (_, parent_root) = store .block_entry(&root) .ok_or(Error::SpecAssert("root in store.blocks"))?; let index = store.block_index(); - let parent_weight = get_attestation_score(store, &index, parent_root, &justified_state)?; + let parent_weight = snapshot_attestation_score(store, &index, parent_root, &balances)?; Ok(parent_weight > parent_threshold) } @@ -3208,16 +3511,15 @@ pub fn should_apply_proposer_boost( if store.proposer_boost_root().is_zero() { return Ok(false); } - let justified_checkpoint = store.beacon_justified_checkpoint(); - let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; + let balances = justified_balances(store, config)?; let index = store.block_index(); should_apply_proposer_boost_with( store, config, committees, &index, - &justified_state, - |parent_root| get_attestation_score(store, &index, parent_root, &justified_state), + &balances, + |parent_root| snapshot_attestation_score(store, &index, parent_root, &balances), ) } @@ -3238,7 +3540,7 @@ fn should_apply_proposer_boost_with( config: &Config, committees: &CommitteeCache, index: &HashMap, - justified_state: &BeaconState, + balances: &JustifiedBalances, parent_score: impl FnOnce(Root) -> Result, ) -> Result { let proposer_boost_root = store.proposer_boost_root(); @@ -3267,7 +3569,7 @@ fn should_apply_proposer_boost_with( parent_slot, config, committees, - justified_state, + balances, parent_attestation_score, )? { return Ok(true); @@ -4377,7 +4679,6 @@ pub fn on_block( .get_state(&parent_root) .expect("get") .ok_or(Error::SpecAssert("block.parent_root in store.block_states"))?; - let mut state = (*parent_state).clone(); // [New in Gloas:EIP7732] If this block builds on its parent's full payload, // that payload must have been verified by `on_execution_payload_envelope`. @@ -4497,8 +4798,25 @@ pub fn on_block( stf::ExecutionEngine::valid() } }; - let transition = - stf::state_transition(&mut state, &signed_block, true, config, &engine, committees); + // + // An epoch-crossing block starts from the precomputed boundary state when + // one is cached, and from the parent's own post-state otherwise. Both reach + // the same post-state: the precomputed one is exactly what `process_slots` + // would have produced from the parent (a gloas parent included: a gloas + // block's post-state never holds its own payload, which only the child + // applies, after `process_slots`, so no second parent state exists). + // Either way `state` is an independent clone, so a failing block cannot + // corrupt a cached entry. + let (mut state, transition) = transition_block( + store, + &parent_state, + parent_root, + &signed_block, + true, + config, + &engine, + committees, + ); // `optimistic-sync.md`: a block deemed `INVALIDATED` MUST NOT be included // in the canonical chain. That is stated here, on the verdict, rather than @@ -4542,6 +4860,11 @@ pub fn on_block( transition?; + // A block at its epoch's first slot is its own epoch-boundary block: its + // post-state is the checkpoint state. Cached at import so the tick that + // justifies it finds the balances built; see `justified_balances`. + cache_first_slot_balances(store, &state, block_root); + // Cache the state root in the latest block header. Sound because the // `true` above means `state_transition` checked it against the root it // computed; see `BeaconState::compute_state_root`. @@ -5363,7 +5686,12 @@ mod tests { /// [`anchor_pair`] over a registry of `count` validators, for the weight /// tests, which name a voter per validator index. fn anchor_pair_with(count: usize) -> (BeaconState, SignedBeaconBlock) { - let mut state = test_state::with_validators(count); + anchor_pair_from(test_state::with_validators(count)) + } + + /// [`anchor_pair`] over a caller-built `state`, for tests that need a + /// registry with slashed or inactive validators in it. + fn anchor_pair_from(mut state: BeaconState) -> (BeaconState, SignedBeaconBlock) { let parent_root = state.latest_block_header().parent_root; let mut signed = block(state.slot(), parent_root); @@ -5412,7 +5740,12 @@ mod tests { /// The anchor is what `checkpoint_state` resolves the justified checkpoint /// to, which is the one thing both weight functions need from a real store. fn anchored_store(count: usize) -> (Store, Root, Slot) { - let (anchor_state, anchor_block) = anchor_pair_with(count); + anchored_store_from(test_state::with_validators(count)) + } + + /// [`anchored_store`] over a caller-built state. + fn anchored_store_from(state: BeaconState) -> (Store, Root, Slot) { + let (anchor_state, anchor_block) = anchor_pair_from(state); let anchor_slot = anchor_state.slot(); let anchor_root = anchor_block.message_hash_tree_root(); let store = get_forkchoice_store( @@ -5425,13 +5758,30 @@ mod tests { (store, anchor_root, anchor_slot) } + /// A `count`-validator state (`count >= 9`) whose registry has every kind + /// of validator the weight paths must tell apart, at the state's own epoch: + /// validator 6 is slashed but active, 7 activates one epoch later, 8 exited + /// at this epoch, and the rest are plain active ones. + fn state_with_slashed_and_inactive_validators(count: usize) -> BeaconState { + let mut state = test_state::with_validators(count); + let epoch = get_current_epoch(&state); + state.validator_mut(6).expect("registered").slashed = true; + state.validator_mut(7).expect("registered").activation_epoch = epoch + 1; + state.validator_mut(8).expect("registered").exit_epoch = epoch; + state.apply_pending_mutations(); + state + } + /// `compute_weights` is the specification's `get_weight` for every root at /// once, so the two have to agree root by root: over a fork, over voters /// spread across both branches, and with the proposer boost applied. #[test] fn the_single_pass_weights_match_the_specifications_per_root_weight() { let config = Config::active(); - let (mut store, anchor_root, anchor_slot) = anchored_store(8); + // Validators 6 (slashed), 7 (not yet active) and 8 (exited) vote below + // and must weigh nothing; validator 40 is past the registry. + let (mut store, anchor_root, anchor_slot) = + anchored_store_from(state_with_slashed_and_inactive_validators(12)); // anchor -> a -> {b, c}: a fork whose two leaves split the vote, so a // wrong fold shows up as a leaf carrying its sibling's balance. @@ -5478,6 +5828,17 @@ mod tests { }, ); store.insert_equivocating_index(5); + for (validator_index, root) in [(6, c_root), (7, c_root), (8, c_root), (40, c_root)] { + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: 0, + root, + payload_present: false, + }, + ); + } store.set_proposer_boost_root(b_root); let weights = compute_weights(&store, &index, &config).expect("the anchor state is there"); @@ -5493,6 +5854,357 @@ mod tests { weights[&b_root] > weights[&c_root], "three voters and the boost must outweigh one voter" ); + assert_eq!( + weights[&c_root], + preset::MAX_EFFECTIVE_BALANCE, + "only validator 3 counts on c: the slashed, inactive, exited and unknown voters weigh nothing" + ); + assert_eq!( + weights[&b_root], + 3 * preset::MAX_EFFECTIVE_BALANCE + + get_proposer_score(&store, &config).expect("the anchor state is there"), + ); + } + + /// The boost's committee weight divides the total active balance, which + /// counts a slashed validator that is still active and leaves out one that + /// is not active yet or already exited: the snapshot's per-vote zeros for + /// the first kind must not leak into the total. + #[test] + fn the_proposer_score_counts_a_slashed_but_active_validator() { + let config = Config::active(); + let (store, _anchor_root, _anchor_slot) = + anchored_store_from(state_with_slashed_and_inactive_validators(12)); + + // 12 validators, 2 of them (7 and 8) inactive; the slashed one stays. + let expected = committee_fraction( + 10 * preset::MAX_EFFECTIVE_BALANCE, + config.proposer_score_boost, + ); + assert_eq!( + get_proposer_score(&store, &config).expect("the anchor state is there"), + expected + ); + assert_ne!( + expected, + committee_fraction( + 9 * preset::MAX_EFFECTIVE_BALANCE, + config.proposer_score_boost + ), + "a total that dropped the slashed validator would differ" + ); + // And it is the state's own definition, not a second one. + let state = checkpoint_state(&store, &store.beacon_justified_checkpoint(), &config) + .expect("the anchor state is there"); + assert_eq!( + expected, + calculate_committee_fraction(&state, config.proposer_score_boost).expect("total"), + ); + } + + #[test] + fn building_the_snapshot_handles_activation_exit_slashing_and_an_empty_set() { + let mut state = test_state::with_validators(6); + let epoch = get_current_epoch(&state); + state.validator_mut(1).expect("registered").activation_epoch = epoch; + state.validator_mut(2).expect("registered").activation_epoch = epoch + 1; + state.validator_mut(3).expect("registered").exit_epoch = epoch; + state.validator_mut(4).expect("registered").slashed = true; + state.apply_pending_mutations(); + let checkpoint = Checkpoint { + epoch, + root: Root::repeat_byte(1), + }; + + let snapshot = build_justified_balances(checkpoint, &state); + let full = preset::MAX_EFFECTIVE_BALANCE; + // Activation at the epoch counts; exit at the epoch does not; a slashed + // but active validator weighs zero as a voter and counts in the total. + assert_eq!( + [0, 1, 2, 3, 4, 5].map(|index| snapshot.get(index)), + [full, full, 0, 0, 0, full] + ); + assert_eq!(snapshot.total_active_balance(), 4 * full); + assert_eq!( + snapshot.total_active_balance(), + get_total_active_balance(&state).expect("total") + ); + assert_eq!(snapshot.get(6), 0, "past the registry reads zero"); + assert_eq!(snapshot.checkpoint(), checkpoint); + + // No active validator at all: the total is floored like the spec's. + for index in 0..6 { + state.validator_mut(index).expect("registered").exit_epoch = epoch; + } + state.apply_pending_mutations(); + let empty = build_justified_balances(checkpoint, &state); + assert_eq!( + empty.total_active_balance(), + preset::EFFECTIVE_BALANCE_INCREMENT + ); + assert_eq!( + empty.total_active_balance(), + get_total_active_balance(&state).expect("total") + ); + assert_eq!(empty.get(0), 0); + } + + #[test] + fn a_new_justified_checkpoint_rebuilds_the_snapshot() { + let config = Config::active(); + let (mut store, anchor_root, _anchor_slot) = anchored_store(8); + + let first = justified_balances(&store, &config).expect("the anchor state is there"); + let again = justified_balances(&store, &config).expect("cached"); + assert!( + Arc::ptr_eq(&first, &again), + "same checkpoint, same snapshot" + ); + assert_eq!(first.get(0), preset::MAX_EFFECTIVE_BALANCE); + + // A later justified checkpoint over a state with different balances, + // planted where `checkpoint_state` finds it. + let next = Checkpoint { + epoch: first.checkpoint().epoch + 1, + root: Root::repeat_byte(0x77), + }; + let mut state = test_state::with_validators(8); + *state.slot_mut() = compute_start_slot_at_epoch(next.epoch); + state + .validator_mut(0) + .expect("registered") + .effective_balance = 5 * preset::EFFECTIVE_BALANCE_INCREMENT; + state.validator_mut(1).expect("registered").slashed = true; + state.apply_pending_mutations(); + store.cache_state( + CacheKey::CheckpointState { + epoch: next.epoch, + root: next.root, + }, + Arc::new(state), + ); + let finalized = store.beacon_finalized_checkpoint(); + update_checkpoints(&mut store, next, finalized); + assert_ne!(store.beacon_justified_checkpoint().root, anchor_root); + + let rebuilt = justified_balances(&store, &config).expect("planted state"); + assert!(!Arc::ptr_eq(&first, &rebuilt)); + assert_eq!(rebuilt.checkpoint(), next); + assert_eq!(rebuilt.get(0), 5 * preset::EFFECTIVE_BALANCE_INCREMENT); + assert_eq!(rebuilt.get(1), 0, "slashed in the new state"); + assert_eq!(rebuilt.get(2), preset::MAX_EFFECTIVE_BALANCE); + } + + /// A checkpoint whose state is nowhere in the store, with the snapshot for + /// it already cached, as block import leaves it. Every head-path reader + /// must answer from the snapshot: any `checkpoint_state` for it fails. + fn store_with_only_a_justified_snapshot( + state: &BeaconState, + ) -> (Store, Checkpoint, Root, CommitteeCache) { + let (mut store, anchor_root, _) = anchored_store_from(state.clone()); + let next = Checkpoint { + epoch: store.beacon_justified_checkpoint().epoch + 1, + root: Root::repeat_byte(0x77), + }; + let finalized = store.beacon_finalized_checkpoint(); + update_checkpoints(&mut store, next, finalized); + store.insert_justified_balances(Arc::new(build_justified_balances(next, state))); + (store, next, anchor_root, CommitteeCache::default()) + } + + #[test] + fn a_snapshot_hit_reads_no_justified_state_on_any_head_path() { + let config = Config::active(); + let state = state_with_slashed_and_inactive_validators(12); + let (store, next, anchor_root, committees) = store_with_only_a_justified_snapshot(&state); + assert!( + checkpoint_state(&store, &next, &config).is_err(), + "the justified state is not anywhere to be found" + ); + + let snapshot = justified_balances(&store, &config).expect("a hit"); + assert_eq!(snapshot.checkpoint(), next); + let index = store.block_index(); + compute_weights(&store, &index, &config).expect("pre-gloas weights"); + for rules in [ForkRules::PreGloas, ForkRules::Gloas] { + compute_node_weights(&store, &index, &config, &committees, rules) + .unwrap_or_else(|err| panic!("{rules:?} head walk: {err:?}")); + } + get_proposer_score(&store, &config).expect("boost score"); + is_head_weak(&store, anchor_root, &config, &committees).expect("is_head_weak"); + assert!(!should_apply_proposer_boost(&store, &config, &committees).expect("no boost root")); + } + + /// The snapshot keeps what `is_head_weak` needs of a slashed equivocator + /// (its raw effective balance, though its vote weighs zero), and the + /// snapshot-based decision equals the specification's state-based one. + #[test] + fn a_slashed_equivocator_still_counts_toward_head_weakness_from_the_snapshot() { + let config = Config::active(); + let mut state = test_state::with_validators(preset::SLOTS_PER_EPOCH as usize); + for index in 0..preset::SLOTS_PER_EPOCH { + state.validator_mut(index).expect("registered").slashed = true; + } + state.apply_pending_mutations(); + let (mut store, anchor_root, anchor_slot) = anchored_store_from(state); + let committees = CommitteeCache::default(); + + // The specification's `is_head_weak`, read off the justified state. + let reference = |store: &Store| -> bool { + let justified_state = + checkpoint_state(store, &store.beacon_justified_checkpoint(), &config) + .expect("justified state"); + let index = store.block_index(); + let mut weight = + get_attestation_score(store, &index, anchor_root, &justified_state).expect("score"); + let head_state = store.get_state(&anchor_root).expect("get").expect("state"); + let epoch_committees = + committees.committees(&head_state, compute_epoch_at_slot(anchor_slot)); + for committee_index in 0..epoch_committees.committees_per_slot() { + for &validator in epoch_committees + .committee(anchor_slot, committee_index) + .expect("committee") + { + if store.is_equivocating(validator) { + weight += justified_state + .validator(validator) + .expect("registered") + .effective_balance; + } + } + } + let threshold = + calculate_committee_fraction(&justified_state, config.reorg_head_weight_threshold) + .expect("total"); + weight < threshold + }; + + assert!(is_head_weak(&store, anchor_root, &config, &committees).unwrap()); + assert_eq!( + is_head_weak(&store, anchor_root, &config, &committees).unwrap(), + reference(&store) + ); + + let head_state = store.get_state(&anchor_root).expect("get").expect("state"); + let equivocator = committees + .committees(&head_state, compute_epoch_at_slot(anchor_slot)) + .committee(anchor_slot, 0) + .expect("committee 0")[0]; + store.insert_equivocating_index(equivocator); + + let snapshot = justified_balances(&store, &config).expect("snapshot"); + assert_eq!(snapshot.get(equivocator), 0, "slashed: no vote weight"); + assert_eq!( + snapshot.effective_balance(equivocator), + preset::MAX_EFFECTIVE_BALANCE, + "but its effective balance is still kept" + ); + assert!(!is_head_weak(&store, anchor_root, &config, &committees).unwrap()); + assert_eq!( + is_head_weak(&store, anchor_root, &config, &committees).unwrap(), + reference(&store) + ); + } + + // ---- balances built at import ---- + + /// Importing past empty first slots makes `(epoch, parent)` an + /// epoch-boundary checkpoint, and the balances cached for it are what + /// `checkpoint_state` for that key yields, on a precompute miss and on a + /// hit alike, with the block's post-state unchanged by the split. + #[test] + fn a_crossing_import_caches_the_boundary_balances_for_the_parent() { + let config = Config::mainnet(); + let parent = last_slot_parent(); + let parent_root = Root::repeat_byte(0x42); + let epoch = 2; + let boundary = Checkpoint { + epoch, + root: parent_root, + }; + let expected = build_justified_balances( + boundary, + &advance_to_epoch_start(&parent, epoch, &config).expect("boundary state"), + ); + let first_slot = compute_start_slot_at_epoch(epoch); + + for slot in [first_slot + 1, first_slot + 3] { + let (block, committed) = block_on(&parent, slot); + let miss_store = empty_store(); + let hit_store = store_with_precompute(&parent, parent_root, epoch); + for (store, label) in [(&miss_store, "miss"), (&hit_store, "hit")] { + assert!(store.justified_balances(&boundary).is_none()); + let root = post_state_root(store, &parent, parent_root, &block); + assert_eq!(root, committed, "slot {slot} {label}: state root verifies"); + let cached = store + .justified_balances(&boundary) + .unwrap_or_else(|| panic!("slot {slot} {label}: boundary balances cached")); + assert_eq!(*cached, expected, "slot {slot} {label}"); + } + } + } + + /// A block exactly at the epoch's first slot is its own boundary block: + /// nothing is cached for its parent, and its post-state's balances are + /// cached under `(epoch, block_root)`, equal to what `checkpoint_state` + /// then derives. + #[test] + fn a_first_slot_import_caches_balances_from_its_own_post_state() { + let config = Config::mainnet(); + let parent = last_slot_parent(); + let parent_root = Root::repeat_byte(0x42); + let block_root = Root::repeat_byte(0x99); + let epoch = 2; + let first_slot = compute_start_slot_at_epoch(epoch); + let (signed, _) = block_on(&parent, first_slot); + + let mut store = empty_store(); + let (mut post, outcome) = transition_block( + &store, + &parent, + parent_root, + &signed, + false, + &config, + &stf::ExecutionEngine::valid(), + &CommitteeCache::default(), + ); + outcome.expect("applies"); + post.apply_pending_mutations(); + assert!( + store + .justified_balances(&Checkpoint { + epoch, + root: parent_root + }) + .is_none(), + "the parent is not this epoch's boundary block" + ); + + cache_first_slot_balances(&store, &post, block_root); + let key = Checkpoint { + epoch, + root: block_root, + }; + let cached = store.justified_balances(&key).expect("cached"); + + store + .insert_signed_block(block_root, block(first_slot, parent_root)) + .expect("insert block"); + store.insert_state(block_root, post).expect("insert state"); + let derived = checkpoint_state(&store, &key, &config).expect("checkpoint state"); + assert_eq!(*cached, build_justified_balances(key, &derived)); + + // Any other slot of the epoch is not a boundary. + let other = Root::repeat_byte(0x55); + let mut later = (*derived).clone(); + stf::process_slots(&mut later, first_slot + 1, &config).expect("advance"); + cache_first_slot_balances(&store, &later, other); + assert!( + store + .justified_balances(&Checkpoint { epoch, root: other }) + .is_none() + ); } /// The failure a live mainnet follower hit: `promote_beacon_anchor` prunes @@ -8966,4 +9678,235 @@ mod tests { ); assert_eq!(store.payload_link(&root), Some(link)); } + + // ---- epoch precompute ---- + + use crate::beacon::block_production::{ + BlockInputs, advance_to_slot, assemble_block, payload_inputs, + }; + use crate::beacon::containers::electra; + use crate::beacon::helpers::accessors::{get_beacon_proposer_index, get_domain}; + use crate::beacon::helpers::test_state::{sign_for, with_signing_validators_at}; + use crate::beacon::primitives::Bytes32; + use ethlambda_types::beacon::containers::deneb::ExecutionPayload; + use ethlambda_types::beacon::containers::electra::{BeaconBlockBody, ExecutionRequests}; + + /// A fulu state at the last slot of epoch 1, the parent of every + /// epoch-crossing block below. + fn last_slot_parent() -> BeaconState { + let mut state = with_signing_validators_at(crate::beacon::ForkName::Fulu, 64); + let lookahead = + crate::beacon::helpers::fulu::initialize_proposer_lookahead(&state).expect("lookahead"); + let sync_committee = + crate::beacon::helpers::altair::get_next_sync_committee(&state).expect("committee"); + let BeaconState::Fulu(inner) = &mut state else { + unreachable!("built as fulu") + }; + inner.proposer_lookahead = lookahead.try_into().expect("lookahead length"); + inner.current_sync_committee = sync_committee.clone(); + inner.next_sync_committee = sync_committee; + let last_slot = 2 * preset::SLOTS_PER_EPOCH - 1; + advance_to_slot(&state, last_slot, &Config::mainnet()).expect("advance") + } + + /// A block at `slot` built on `parent` the way a proposer would, and the + /// post-state root it commits to. + fn block_on(parent: &BeaconState, slot: Slot) -> (SignedBeaconBlock, Root) { + let config = Config::mainnet(); + let advanced = advance_to_slot(parent, slot, &config).expect("advance"); + let proposer = get_beacon_proposer_index(&advanced).expect("proposer"); + let epoch = get_current_epoch(&advanced); + let domain = get_domain(&advanced, constants::DOMAIN_RANDAO, Some(epoch)); + let randao_reveal: BlsSignature = sign_for( + proposer as usize, + compute_signing_root(epoch.hash_tree_root(), domain), + ); + let payload = payload_inputs(&advanced, &config).expect("payload inputs"); + let inputs = BlockInputs { + randao_reveal, + graffiti: Bytes32::ZERO, + attestations: Vec::new(), + execution_payload: ExecutionPayload { + parent_hash: payload.parent_hash, + prev_randao: payload.prev_randao, + timestamp: payload.timestamp, + withdrawals: payload.withdrawals.try_into().expect("withdrawals"), + ..BeaconBlockBody::empty().execution_payload + }, + blob_kzg_commitments: Vec::new(), + execution_requests: ExecutionRequests::default(), + }; + let message = assemble_block(&advanced, inputs, &config).expect("assemble"); + let state_root = message.state_root; + let signed = SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message, + signature: BlsSignature::default(), + }); + (signed, state_root) + } + + fn post_state_root( + store: &Store, + parent: &BeaconState, + parent_root: Root, + block: &SignedBeaconBlock, + ) -> Root { + let (mut state, outcome) = transition_block( + store, + parent, + parent_root, + block, + false, + &Config::mainnet(), + &stf::ExecutionEngine::valid(), + &CommitteeCache::default(), + ); + outcome.expect("the block applies"); + state.apply_pending_mutations(); + state.hash_tree_root() + } + + fn store_with_precompute(parent: &BeaconState, parent_root: Root, epoch: Epoch) -> Store { + let store = empty_store(); + let precomputed = + advance_to_epoch_start(parent, epoch, &Config::mainnet()).expect("precompute"); + store.cache_state( + CacheKey::CheckpointState { + epoch, + root: parent_root, + }, + Arc::new(precomputed), + ); + store + } + + /// Resuming from the precomputed state is not a different transition: the + /// post-state is the one the inline path commits to, for a block on the + /// boundary and for one past a skipped first slot. + #[test] + fn resuming_from_a_precomputed_state_gives_the_inline_post_state() { + let parent = last_slot_parent(); + let parent_root = Root::repeat_byte(0x42); + let epoch = 2; + let first_slot = compute_start_slot_at_epoch(epoch); + + for slot in [first_slot, first_slot + 1, first_slot + 3] { + let (block, committed) = block_on(&parent, slot); + let hit_store = store_with_precompute(&parent, parent_root, epoch); + assert!( + precomputed_epoch_state(&hit_store, parent.slot(), parent_root, slot).is_some() + ); + + let inline = post_state_root(&empty_store(), &parent, parent_root, &block); + let resumed = post_state_root(&hit_store, &parent, parent_root, &block); + assert_eq!(inline, committed, "slot {slot}: inline matches the block"); + assert_eq!(resumed, inline, "slot {slot}: resumed matches inline"); + } + } + + #[test] + fn only_an_epoch_crossing_import_looks_for_a_precomputed_state() { + let parent = last_slot_parent(); + let parent_root = Root::repeat_byte(0x42); + let store = store_with_precompute(&parent, parent_root, 2); + let first_slot = compute_start_slot_at_epoch(2); + + // Crosses: parent before the boundary, block at or after it. + assert!(precomputed_epoch_state(&store, parent.slot(), parent_root, first_slot).is_some()); + // Same epoch as its parent: nothing to resume. + assert!(precomputed_epoch_state(&store, first_slot, parent_root, first_slot + 1).is_none()); + // A different parent has no entry. + assert!( + precomputed_epoch_state(&store, parent.slot(), Root::repeat_byte(1), first_slot) + .is_none() + ); + } + + /// The worker's result is what `checkpoint_state` derives for the same + /// `(epoch, root)`, which is why one cache key can serve both. + #[test] + fn the_precompute_equals_checkpoint_state_for_the_same_key() { + let config = Config::mainnet(); + let parent = last_slot_parent(); + let root = Root::repeat_byte(0x42); + let mut store = empty_store(); + store + .insert_signed_block(root, block(parent.slot(), Root::ZERO)) + .expect("insert block"); + store + .insert_state(root, parent.clone()) + .expect("insert state"); + let epoch = 2; + + let worker = advance_to_epoch_start(&parent, epoch, &config).expect("precompute"); + let derived = + checkpoint_state(&store, &Checkpoint { epoch, root }, &config).expect("checkpoint"); + + assert_eq!(worker.slot(), compute_start_slot_at_epoch(epoch)); + assert_eq!(worker.slot(), derived.slot()); + assert_eq!(worker.hash_tree_root(), derived.hash_tree_root()); + } + + /// A gloas state at the last slot of epoch 1, the parent of the gloas + /// epoch-crossing cases below. + fn gloas_last_slot_parent() -> BeaconState { + let state = test_state::with_validators_at(crate::beacon::ForkName::Gloas, 64); + let last_slot = 2 * preset::SLOTS_PER_EPOCH - 1; + advance_to_slot(&state, last_slot, &Config::mainnet()).expect("advance") + } + + /// A gloas block's post-state never holds its own payload (the child applies + /// it inside `process_block`, after `process_slots`), and the store keeps + /// one state per root, so the precompute starts from exactly the state an + /// import of a child would. Checked three ways for a gloas parent: the + /// worker's result is `checkpoint_state`'s for the same key, an import + /// finds it under that key, and advancing past a skipped first slot from it + /// gives the state a plain `process_slots` from the parent gives. + #[test] + fn a_gloas_precompute_starts_from_the_state_an_import_would() { + let config = Config::mainnet(); + let parent = gloas_last_slot_parent(); + let root = Root::repeat_byte(0x42); + let mut store = empty_store(); + store + .insert_signed_block(root, block(parent.slot(), Root::ZERO)) + .expect("insert block"); + store + .insert_state(root, parent.clone()) + .expect("insert state"); + let epoch = 2; + let first_slot = compute_start_slot_at_epoch(epoch); + + let worker = advance_to_epoch_start(&parent, epoch, &config).expect("precompute"); + let derived = + checkpoint_state(&store, &Checkpoint { epoch, root }, &config).expect("checkpoint"); + assert_eq!(worker.fork_name(), crate::beacon::ForkName::Gloas); + assert_eq!(worker.hash_tree_root(), derived.hash_tree_root()); + + store.cache_state(CacheKey::CheckpointState { epoch, root }, Arc::new(worker)); + let stored = store + .get_state(&root) + .expect("get") + .expect("the parent's own post-state"); + assert!( + precomputed_epoch_state(&store, stored.slot(), root, first_slot).is_some(), + "an import of a child of `root` finds the entry" + ); + + for slot in [first_slot + 1, first_slot + 3] { + let mut resumed = (*precomputed_epoch_state(&store, stored.slot(), root, slot) + .expect("cached")) + .clone(); + stf::process_slots(&mut resumed, slot, &config).expect("advance the rest"); + let mut inline = (*stored).clone(); + stf::process_slots(&mut inline, slot, &config).expect("advance inline"); + resumed.apply_pending_mutations(); + inline.apply_pending_mutations(); + assert_eq!( + resumed.hash_tree_root(), + inline.hash_tree_root(), + "slot {slot}" + ); + } + } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs index ea23506a..71ce8fe9 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs @@ -136,7 +136,33 @@ pub fn state_transition( committees: &CommitteeCache, ) -> Result<()> { process_slots(state, signed_block.slot(), config)?; + apply_block( + state, + signed_block, + validate_result, + config, + engine, + committees, + ) +} +/// [`state_transition`] from the point where `state` is already at the block's +/// own slot: everything after `process_slots`. +/// +/// For a caller that advanced the state itself (fork choice resuming from a +/// precomputed epoch-boundary state). `process_slots` rejects a target equal to +/// the current slot, so such a state cannot go through [`state_transition`] +/// when the block sits exactly on the boundary. A state at the wrong slot is +/// still refused: the block header check in `process_block` compares the +/// block's slot with the state's. +pub fn apply_block( + state: &mut BeaconState, + signed_block: &containers::SignedBeaconBlock, + validate_result: bool, + config: &Config, + engine: &ExecutionEngine, + committees: &CommitteeCache, +) -> Result<()> { // After `process_slots`, never before it. A block proposed at the first slot // of a fork's activation epoch is the *post*-fork shape while the state // arriving here is still the pre-fork one, which is exactly the case a fork diff --git a/crates/blockchain/state_transition/src/metrics.rs b/crates/blockchain/state_transition/src/metrics.rs index e7da0ce8..6b16d7c3 100644 --- a/crates/blockchain/state_transition/src/metrics.rs +++ b/crates/blockchain/state_transition/src/metrics.rs @@ -62,6 +62,88 @@ pub fn inc_committee_cache_lookups(result: &str) { .inc(); } +static LEAN_BEACON_JUSTIFIED_BALANCES_LOOKUPS_TOTAL: LazyLock = LazyLock::new( + || { + register_int_counter_vec!( + "lean_beacon_justified_balances_lookups_total", + "Beacon fork-choice justified-balances snapshot lookups, by whether the cached snapshot served them", + &["result"] + ) + .unwrap() + }, +); + +/// Count one justified-balances lookup: `hit` (the cached snapshot matched the +/// justified checkpoint) or `miss` (it was rebuilt from the checkpoint state). +pub fn inc_justified_balances_lookups(result: &str) { + LEAN_BEACON_JUSTIFIED_BALANCES_LOOKUPS_TOTAL + .with_label_values(&[result]) + .inc(); +} + +static LEAN_BEACON_JUSTIFIED_BALANCES_BUILD_SECONDS: LazyLock = LazyLock::new(|| { + register_histogram!( + "lean_beacon_justified_balances_build_seconds", + "Duration of one justified-balances snapshot build (one pass over the checkpoint state's registry)", + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0] + ) + .unwrap() +}); + +/// Time one justified-balances snapshot build, excluding the checkpoint state +/// lookup it starts from. +pub fn time_justified_balances_build() -> TimingGuard { + TimingGuard::new(&LEAN_BEACON_JUSTIFIED_BALANCES_BUILD_SECONDS) +} + +static LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_epoch_precompute_lookups_total", + "Epoch-crossing block imports, by whether a precomputed epoch-boundary state was cached", + &["result"] + ) + .unwrap() +}); + +/// Count one epoch-crossing import's lookup of a precomputed state: `hit` or +/// `miss`. +pub fn inc_epoch_precompute_lookups(result: &str) { + LEAN_BEACON_EPOCH_PRECOMPUTE_LOOKUPS_TOTAL + .with_label_values(&[result]) + .inc(); +} + +static LEAN_BEACON_EPOCH_PRECOMPUTE_SECONDS: LazyLock = LazyLock::new(|| { + register_histogram!( + "lean_beacon_epoch_precompute_seconds", + "Duration of one epoch-transition precompute: advance, flush and hash", + vec![0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0, 8.0] + ) + .unwrap() +}); + +/// Start timing one epoch precompute. Records duration when the guard is dropped. +pub fn time_epoch_precompute() -> TimingGuard { + TimingGuard::new(&LEAN_BEACON_EPOCH_PRECOMPUTE_SECONDS) +} + +static LEAN_BEACON_EPOCH_PRECOMPUTE_STARTED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter_vec!( + "lean_beacon_epoch_precompute_started_total", + "Epoch precompute workers started, by what triggered them", + &["trigger"] + ) + .unwrap() +}); + +/// Count one precompute worker started: `head` (a last-slot block became head) +/// or `timer` (three quarters into a last slot with nothing computed). +pub fn inc_epoch_precompute_started(trigger: &str) { + LEAN_BEACON_EPOCH_PRECOMPUTE_STARTED_TOTAL + .with_label_values(&[trigger]) + .inc(); +} + static LEAN_STATE_TRANSITION_TIME_SECONDS: LazyLock = LazyLock::new(|| { register_histogram!( "lean_state_transition_time_seconds", diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index fb901a2e..4c2f4ea7 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -976,6 +976,108 @@ pub(super) fn apply_checks( if let Some(expected) = &checks.get_proposer_head { check_get_proposer_head(expected, store, config, committees)?; } + check_weights_against_the_spec(store, config)?; + check_node_weights_against_the_spec(store, config, committees) +} + +/// The gloas counterpart of [`check_weights_against_the_spec`], run at every +/// `checks` step of a case whose current slot is gloas. +/// +/// The head walk weighs payload-aware nodes with `compute_node_weights`, whose +/// vote loop reads the justified-balances snapshot; `gloas_get_weight` is the +/// specification's per-node definition (payload status, the previous-slot +/// rule, the gated proposer boost) and reads the justified checkpoint state +/// itself. Every node under the justified root is compared, not only the +/// leaves the `viable_for_head_roots_and_weights` check covers. +fn check_node_weights_against_the_spec( + store: &Store, + config: &Config, + committees: &CommitteeCache, +) -> Result<(), String> { + let current_slot = fork_choice::get_current_slot(store, config); + let fork = config.fork_at_epoch(current_slot / preset::SLOTS_PER_EPOCH); + if ForkRules::of(fork) != ForkRules::Gloas { + return Ok(()); + } + let index = store.block_index(); + let blocks = fork_choice::get_filtered_block_tree(store, &index, config) + .map_err(|err| format!("get_filtered_block_tree: {err:?}"))?; + let walked = + fork_choice::compute_node_weights(store, &index, config, committees, ForkRules::Gloas) + .map_err(|err| format!("compute_node_weights: {err:?}"))?; + let mut pending = vec![ForkChoiceNode { + root: store.beacon_justified_checkpoint().root, + payload_status: PayloadStatus::Pending, + }]; + while let Some(node) = pending.pop() { + let spec = match fork_choice::gloas_get_weight(store, node, config, committees) { + Ok(spec) => Some(spec), + // A vote for a block that invalidation removed from the index: the + // spec-literal weight raises on it and the walk drops it on + // purpose, so the two are not comparable there. + Err(err) if format!("{err:?}").contains("root in store.blocks") => None, + Err(err) => { + return Err(format!( + "gloas_get_weight(0x{}, {}): {err:?}", + hex::encode(node.root.0), + node.payload_status as u8 + )); + } + }; + if let Some(spec) = spec { + let block_slot = index + .get(&node.root) + .map(|&(slot, _)| slot) + .ok_or("a tree node is not in the block index")?; + let single_pass = walked.weight(node, block_slot); + if spec != single_pass { + return Err(format!( + "weight of 0x{} ({}): spec gloas_get_weight {spec}, compute_node_weights {single_pass}", + hex::encode(node.root.0), + node.payload_status as u8 + )); + } + } + pending.extend( + fork_choice::get_node_children(store, &blocks, node) + .map_err(|err| format!("get_node_children: {err:?}"))?, + ); + } + Ok(()) +} + +/// Oracle for the fork-choice weights, run at every `checks` step. +/// +/// No fixture checks weights directly, but `get_head` descends on +/// `compute_weights` (one pass over the votes, balances from the justified- +/// balances snapshot), while `get_weight` is the specification's per-root +/// definition and reads the justified checkpoint state itself. Comparing the +/// two for every block in the filtered tree puts the snapshot against the spec +/// path on every fixture, whatever the fixture asserts. +fn check_weights_against_the_spec(store: &Store, config: &Config) -> Result<(), String> { + let index = store.block_index(); + let tree = fork_choice::get_filtered_block_tree(store, &index, config) + .map_err(|err| format!("get_filtered_block_tree: {err:?}"))?; + let weights = fork_choice::compute_weights(store, &index, config) + .map_err(|err| format!("compute_weights: {err:?}"))?; + for root in tree.keys() { + let spec = match fork_choice::get_weight(store, &index, *root, config) { + Ok(spec) => spec, + // A vote for a block that invalidation removed from the index + // (`sync/optimistic`): the specification's `get_weight` raises on + // it, and `compute_weights` drops such a vote on purpose, so the + // two are not comparable for that case. + Err(err) if format!("{err:?}").contains("root in store.blocks") => continue, + Err(err) => return Err(format!("get_weight(0x{}): {err:?}", hex::encode(root.0))), + }; + let single_pass = weights.get(root).copied().unwrap_or_default(); + if spec != single_pass { + return Err(format!( + "weight of 0x{}: spec get_weight {spec}, compute_weights {single_pass}", + hex::encode(root.0) + )); + } + } Ok(()) } diff --git a/crates/common/types/src/beacon/fork_choice.rs b/crates/common/types/src/beacon/fork_choice.rs index c39486a2..96266153 100644 --- a/crates/common/types/src/beacon/fork_choice.rs +++ b/crates/common/types/src/beacon/fork_choice.rs @@ -20,7 +20,10 @@ use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Slot, Uint256}; +use crate::beacon::containers::Checkpoint; +use crate::beacon::primitives::{ + Epoch, ExecutionBlockHash, Gwei, Root, Slot, Uint256, ValidatorIndex, +}; /// One validator's most recent attestation: the epoch it targeted, and the /// block it attested to (the LMD GHOST vote). @@ -206,12 +209,115 @@ pub struct PayloadStatusV1 { pub validation_error: Option, } +/// The justified checkpoint state's balances, flattened for the fork-choice +/// vote loop: `store.checkpoint_states[checkpoint]` as `get_weight` and +/// `get_proposer_score` read it. +/// +/// A tree descent per vote (`state.validator(i)`) is an order of magnitude +/// dearer than an array read, and the loop runs once per `get_head` over every +/// voter. This is built once per justified checkpoint and keyed by it, so a +/// reader compares [`Self::checkpoint`] with the store's current one and +/// rebuilds on a mismatch, with no hook on the code that moves the checkpoint. +/// +/// Held in the store, hence here rather than in `ethlambda-state-transition`; +/// the builder lives there, next to the state accessors. +#[derive(Debug, PartialEq, Eq)] +pub struct JustifiedBalances { + checkpoint: Checkpoint, + /// Zero unless the validator is active at `checkpoint.epoch` and unslashed, + /// the two conditions under which a vote weighs anything. + balances: Box<[Gwei]>, + /// `get_total_active_balance` of the checkpoint state: unlike `balances` + /// it counts slashed validators, and it is floored at one increment. + total_active_balance: Gwei, + /// Every validator's raw effective balance in `EFFECTIVE_BALANCE_INCREMENT` + /// units, whether or not it is active or slashed. Exact, since an + /// effective balance is always a whole number of increments and the + /// largest (`MAX_EFFECTIVE_BALANCE_ELECTRA`) is far below `u16::MAX` + /// increments; two bytes a validator instead of eight. + /// + /// `balances` zeroes a slashed validator, so it cannot answer the one + /// question that needs the raw figure: `is_head_weak` adds back the + /// effective balance of every equivocating validator in the head slot's + /// committees, and an equivocator is often slashed. + effective_increments: Box<[u16]>, +} + +impl JustifiedBalances { + /// Wraps already-computed values; see the field docs for what they mean. + pub fn new( + checkpoint: Checkpoint, + balances: Box<[Gwei]>, + total_active_balance: Gwei, + effective_increments: Box<[u16]>, + ) -> Self { + Self { + checkpoint, + balances, + total_active_balance, + effective_increments, + } + } + + /// `index`'s raw effective balance at the checkpoint, active and unslashed + /// or not; zero past the end, since that validator did not exist then. + pub fn effective_balance(&self, index: ValidatorIndex) -> Gwei { + usize::try_from(index) + .ok() + .and_then(|index| self.effective_increments.get(index)) + .map_or(0, |&increments| { + Gwei::from(increments) * crate::beacon::preset::EFFECTIVE_BALANCE_INCREMENT + }) + } + + /// The checkpoint these balances were derived from. + pub fn checkpoint(&self) -> Checkpoint { + self.checkpoint + } + + /// The weight of `index`'s vote: zero when inactive or slashed, and also + /// past the end, since that validator did not exist at the checkpoint. + pub fn get(&self, index: ValidatorIndex) -> Gwei { + usize::try_from(index) + .ok() + .and_then(|index| self.balances.get(index)) + .copied() + .unwrap_or_default() + } + + /// The checkpoint state's total active balance, as the specification's + /// `get_total_active_balance` defines it. + pub fn total_active_balance(&self) -> Gwei { + self.total_active_balance + } +} + #[cfg(test)] mod tests { use libssz::{SszDecode as _, SszEncode as _}; use super::*; + #[test] + fn justified_balances_read_zero_past_the_registry() { + let balances = JustifiedBalances::new( + Checkpoint::default(), + vec![7, 0, 9].into(), + 16, + vec![1, 2, 3].into(), + ); + assert_eq!(balances.get(0), 7); + assert_eq!(balances.get(1), 0); + assert_eq!(balances.get(2), 9); + assert_eq!(balances.get(3), 0); + assert_eq!(balances.get(u64::MAX), 0); + assert_eq!(balances.total_active_balance(), 16); + let increment = crate::beacon::preset::EFFECTIVE_BALANCE_INCREMENT; + assert_eq!(balances.effective_balance(1), 2 * increment); + assert_eq!(balances.effective_balance(3), 0); + assert_eq!(balances.effective_balance(u64::MAX), 0); + } + #[test] fn a_pow_block_round_trips_through_ssz() { // The store persists these, so the derive has to survive the move out diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 49078d61..412d788f 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -24,8 +24,8 @@ use ethlambda_types::{ }, fork::ForkName, fork_choice::{ - BlockPayloadLink, LatestMessage, PayloadStatus, PayloadStatusEnum, PayloadStatusV1, - PowBlock, + BlockPayloadLink, JustifiedBalances, LatestMessage, PayloadStatus, PayloadStatusEnum, + PayloadStatusV1, PowBlock, }, preset::{PTC_SIZE, Preset, SLOTS_PER_EPOCH}, primitives::ExecutionBlockHash, @@ -647,6 +647,73 @@ impl GossipSignatureBuffer { } } +/// Bound on a validator index [`Store::set_latest_message`] accepts in debug +/// builds. Far above any registry this client will meet, far below what a +/// corrupt index would need to exhaust memory: the dense vote table has one slot +/// per index up to the highest seen. +const MAX_PLAUSIBLE_VALIDATOR_INDEX: u64 = 1 << 26; + +/// How many epoch-boundary [`JustifiedBalances`] snapshots the store keeps. +const JUSTIFIED_BALANCES_CAPACITY: usize = 8; + +/// A small cache of [`JustifiedBalances`], keyed by their checkpoint. +/// +/// Filled at block import, for the epoch-boundary checkpoint a block makes +/// available, so that the tick which moves the justified checkpoint finds the +/// snapshot already built and the head computation never has to rebuild a +/// state. Lighthouse does the same with a plain 4-entry FIFO. Eviction here is +/// smarter, since a network with many forks (several blocks crossing one +/// boundary from different parents) could push the canonical entry out of a +/// FIFO within one epoch: +/// +/// 1. when full, the entry with the lowest epoch goes first (ties: the oldest +/// insert), because an old boundary is the least likely to be justified; +/// 2. the entry for the store's current justified checkpoint is never evicted. +#[derive(Default)] +pub(crate) struct JustifiedBalancesCache { + /// Insert sequence number (for tie-breaking) and the snapshot. + entries: Vec<(u64, Arc)>, + next_sequence: u64, +} + +impl JustifiedBalancesCache { + fn get(&self, checkpoint: &BeaconCheckpoint) -> Option> { + self.entries + .iter() + .find(|(_, balances)| balances.checkpoint() == *checkpoint) + .map(|(_, balances)| Arc::clone(balances)) + } + + /// Adds `balances`, evicting by the policy above if the cache is full. + /// `current_justified` is the entry that must survive. + fn insert(&mut self, balances: Arc, current_justified: &BeaconCheckpoint) { + let sequence = self.next_sequence; + self.next_sequence += 1; + let checkpoint = balances.checkpoint(); + if let Some(entry) = self + .entries + .iter_mut() + .find(|(_, existing)| existing.checkpoint() == checkpoint) + { + *entry = (sequence, balances); + return; + } + if self.entries.len() >= JUSTIFIED_BALANCES_CAPACITY { + let victim = self + .entries + .iter() + .enumerate() + .filter(|(_, (_, entry))| entry.checkpoint() != *current_justified) + .min_by_key(|(_, (sequence, entry))| (entry.checkpoint().epoch, *sequence)) + .map(|(index, _)| index); + if let Some(victim) = victim { + self.entries.swap_remove(victim); + } + } + self.entries.push((sequence, balances)); + } +} + /// Beacon fork-choice state that is per-slot or per-epoch scratch rather than /// chain history: apart from the gloas entries named below, nothing here /// survives a restart, and nothing here is worth the write amplification of @@ -703,7 +770,14 @@ pub(crate) struct BeaconScratch { /// the same deadline it always checked. pub(crate) block_timeliness: HashMap, pub(crate) equivocating_indices: HashSet, - pub(crate) latest_messages: HashMap, + /// Dense, indexed by validator index: the vote loop reads votes and + /// snapshot balances in index order, not hash order. `None` is a validator + /// that has not voted (or an index below the highest one seen). + pub(crate) latest_messages: Vec>, + /// Epoch-boundary balances, flattened for the vote loop + /// and keyed by their own checkpoint. A derived cache: a miss is rebuilt + /// from `checkpoint_states`, so nothing needs to persist it. + pub(crate) justified_balances: JustifiedBalancesCache, pub(crate) pow_blocks: HashMap, pub(crate) unrealized_justifications: HashMap, /// Gloas: beacon block roots whose execution payload envelope has been @@ -3692,17 +3766,28 @@ impl Store { .lock() .unwrap() .latest_messages - .get(&index) + .get(usize::try_from(index).ok()?) .copied() + .flatten() } /// Records the latest attestation for validator `index`. + /// + /// The table grows to hold `index`, so an index must come from a validated + /// attestation (a member of a committee of the registry), never from raw + /// input: a wild index would allocate a table to match. The assertion is + /// the tripwire for that in debug builds and fixtures. pub fn set_latest_message(&mut self, index: u64, message: LatestMessage) { - self.beacon - .lock() - .unwrap() - .latest_messages - .insert(index, message); + debug_assert!( + index < MAX_PLAUSIBLE_VALIDATOR_INDEX, + "validator index {index} would grow the dense vote table far past any registry" + ); + let slot = usize::try_from(index).expect("validator index fits in usize"); + let mut beacon = self.beacon.lock().unwrap(); + if beacon.latest_messages.len() <= slot { + beacon.latest_messages.resize(slot + 1, None); + } + beacon.latest_messages[slot] = Some(message); } /// Gloas: the payload timeliness committee's per-member votes on whether @@ -3749,7 +3834,8 @@ impl Store { } /// Calls `f` with `(validator_index, latest_message)` for every latest - /// message whose validator has not been observed equivocating. + /// message whose validator has not been observed equivocating, in + /// ascending validator index. /// /// Takes a closure rather than returning an iterator or a cloned map: /// the data lives behind a mutex, so a borrow of it cannot escape the @@ -3758,13 +3844,50 @@ impl Store { /// let it count for either side of the fork it created. pub fn for_each_non_equivocating_latest_message(&self, mut f: impl FnMut(u64, LatestMessage)) { let beacon = self.beacon.lock().unwrap(); - for (&index, &message) in &beacon.latest_messages { - if !beacon.equivocating_indices.contains(&index) { - f(index, message); + // Most of the time nobody has equivocated, so skip the set probe per vote. + let any_equivocators = !beacon.equivocating_indices.is_empty(); + for (index, message) in beacon.latest_messages.iter().enumerate() { + let Some(message) = message else { + continue; + }; + let index = index as u64; + if any_equivocators && beacon.equivocating_indices.contains(&index) { + continue; } + f(index, *message); } } + /// The cached balances snapshot for exactly `checkpoint`, if there is one. + /// + /// The checkpoint is the whole key, so a caller never has to know which + /// code moved the justified checkpoint: an unknown checkpoint just misses. + pub fn justified_balances( + &self, + checkpoint: &BeaconCheckpoint, + ) -> Option> { + self.beacon + .lock() + .unwrap() + .justified_balances + .get(checkpoint) + } + + /// Adds a balances snapshot to the cache, replacing one for the same + /// checkpoint. See [`JustifiedBalancesCache`] for the eviction policy. + /// + /// Takes `&self`, like [`Self::cache_state`]: the read-only fork-choice + /// helpers fill it on a miss. + pub fn insert_justified_balances(&self, balances: Arc) { + // Read before taking the lock: it is the one entry eviction must keep. + let current_justified = self.beacon_justified_checkpoint(); + self.beacon + .lock() + .unwrap() + .justified_balances + .insert(balances, ¤t_justified); + } + /// Looks up a PoW block by its own hash, standing in for the /// specification's `get_pow_block(hash)`. pub fn beacon_pow_block(&self, hash: H256) -> Option { @@ -7639,6 +7762,33 @@ mod tests { assert_eq!(seen, vec![1]); } + #[test] + fn latest_messages_are_visited_in_validator_index_order_and_gaps_are_skipped() { + let mut store = Store::test_store(); + let message = |epoch| LatestMessage { + epoch, + slot: epoch * SLOTS_PER_EPOCH, + root: H256::from([epoch as u8; 32]), + payload_present: false, + }; + + // Written out of order, with a gap, and one overwritten. + store.set_latest_message(9, message(1)); + store.set_latest_message(2, message(2)); + store.set_latest_message(5, message(3)); + store.set_latest_message(5, message(4)); + + let mut seen = Vec::new(); + store + .for_each_non_equivocating_latest_message(|index, vote| seen.push((index, vote.epoch))); + assert_eq!(seen, vec![(2, 2), (5, 4), (9, 1)]); + + assert_eq!(store.latest_message(5).map(|vote| vote.epoch), Some(4)); + assert_eq!(store.latest_message(3), None, "a gap is not a vote"); + assert_eq!(store.latest_message(1_000), None, "past the table"); + assert_eq!(store.latest_message(u64::MAX), None); + } + #[test] fn a_pow_block_is_looked_up_by_its_own_hash() { let mut store = Store::test_store(); @@ -8408,4 +8558,89 @@ mod tests { assert!(index.contains_key(&kept)); assert!(!index.contains_key(&removed)); } + + // ---- justified-balances cache ---- + + fn snapshot(epoch: u64, root_byte: u8) -> Arc { + Arc::new(JustifiedBalances::new( + BeaconCheckpoint { + epoch, + root: H256::repeat_byte(root_byte), + }, + vec![1].into(), + 1, + vec![1].into(), + )) + } + + fn cached(cache: &JustifiedBalancesCache, snapshot: &JustifiedBalances) -> bool { + cache.get(&snapshot.checkpoint()).is_some() + } + + #[test] + fn the_balances_cache_is_bounded_and_evicts_the_lowest_epoch_first() { + let mut cache = JustifiedBalancesCache::default(); + let current = snapshot(100, 0); + // Inserted in an order that is not epoch order, so "lowest epoch" + // differs from "oldest insert". + let epochs = [5, 3, 9, 4, 8, 7, 6, 10]; + let entries: Vec<_> = epochs.iter().map(|&epoch| snapshot(epoch, 1)).collect(); + for entry in &entries { + cache.insert(Arc::clone(entry), ¤t.checkpoint()); + } + assert_eq!(cache.entries.len(), JUSTIFIED_BALANCES_CAPACITY); + + let newest = snapshot(11, 1); + cache.insert(Arc::clone(&newest), ¤t.checkpoint()); + assert_eq!(cache.entries.len(), JUSTIFIED_BALANCES_CAPACITY); + assert!(cached(&cache, &newest)); + assert!(!cached(&cache, &entries[1]), "epoch 3, the lowest, is gone"); + assert!(cached(&cache, &entries[3]), "epoch 4 survives for now"); + + cache.insert(snapshot(12, 1), ¤t.checkpoint()); + assert!(!cached(&cache, &entries[3]), "then epoch 4"); + assert_eq!(cache.entries.len(), JUSTIFIED_BALANCES_CAPACITY); + } + + #[test] + fn equal_epochs_evict_the_oldest_insert_first() { + let mut cache = JustifiedBalancesCache::default(); + let current = snapshot(100, 0); + let forks: Vec<_> = (0..JUSTIFIED_BALANCES_CAPACITY as u8) + .map(|byte| snapshot(7, byte)) + .collect(); + for fork in &forks { + cache.insert(Arc::clone(fork), ¤t.checkpoint()); + } + cache.insert(snapshot(7, 200), ¤t.checkpoint()); + assert!(!cached(&cache, &forks[0]), "the oldest of the tie goes"); + assert!(cached(&cache, &forks[1])); + } + + #[test] + fn the_current_justified_entry_is_never_evicted() { + let mut cache = JustifiedBalancesCache::default(); + // The lowest epoch of all, and the one that must survive. + let current = snapshot(1, 0); + cache.insert(Arc::clone(¤t), ¤t.checkpoint()); + for epoch in 2..2 + 3 * JUSTIFIED_BALANCES_CAPACITY as u64 { + cache.insert(snapshot(epoch, 1), ¤t.checkpoint()); + assert!(cached(&cache, ¤t), "after inserting epoch {epoch}"); + assert!(cache.entries.len() <= JUSTIFIED_BALANCES_CAPACITY); + } + } + + #[test] + fn inserting_a_known_checkpoint_replaces_it_without_evicting() { + let mut cache = JustifiedBalancesCache::default(); + let current = snapshot(100, 0); + for epoch in 0..JUSTIFIED_BALANCES_CAPACITY as u64 { + cache.insert(snapshot(epoch, 1), ¤t.checkpoint()); + } + let again = snapshot(0, 1); + cache.insert(Arc::clone(&again), ¤t.checkpoint()); + assert_eq!(cache.entries.len(), JUSTIFIED_BALANCES_CAPACITY); + let found = cache.get(&again.checkpoint()).expect("still cached"); + assert!(Arc::ptr_eq(&found, &again), "the new snapshot replaced it"); + } } diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index a6ee43e5..8c402ada 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -467,6 +467,27 @@ Two new handlers, `on_execution_payload_envelope` and `on_payload_attestation_message`, join the spec's list of ways to change the store. +Fork choice has the same problem on a longer loop: `get_head` weighs every +validator's latest vote by the voter's balance at the justified checkpoint, and +the boost needs that state's total active balance. Both read one +`JustifiedBalances` snapshot (`ethlambda-types`, held in the store's +`BeaconScratch`) instead of a `validator(i)` descent per vote and a registry +scan per boost. It is built from `checkpoint_state(justified)` only, keyed by +the checkpoint it came from, so a moved justified checkpoint is a miss with no +hook on the writers; it holds a zero for validators that are inactive or +slashed, and a separate total that still counts slashed-but-active validators +(the specification's `get_total_active_balance`, floored at one increment). +Equivocations stay out of it: they are store-level and can change at any time, +so they are filtered per vote. `get_weight` stays as the specification's +per-root definition, and the fork-choice fixture runner checks +`compute_weights` against it at every `checks` step. + +The latest votes are stored the same way, as a dense table indexed by validator +index rather than a hash map, so the vote loop reads votes and snapshot +balances in index order. The table grows to the highest voting index, which is +why `set_latest_message` only takes indices from validated attestations (a +debug assertion bounds it). + ## Macros and traits Two `macro_rules!` in the whole crate, both local, both replacing boilerplate that @@ -535,6 +556,35 @@ eleven cores busy to about eight. The 3.3x understates the hashing change, because the later run does strictly more work: `transition` went from failing immediately to running every case. +## Epoch-transition precompute + +The first block of an epoch used to run `process_epoch` inline on the import +path (`fork_choice::on_block`, then `process_slots`), followed by the rehash +the state-root check needs. The beacon chain actor now does that work ahead of +time, on a blocking worker, and the import resumes from the result. + +| Step | What happens | +| --- | --- | +| Trigger: head | An import leaves a block at the last slot of epoch `E` as head: precompute `(E+1, head_root)`. Covers a late last-slot block. | +| Trigger: timer | Three quarters into the last slot of `E` (derived from the configured slot duration), if nothing for the current head is cached or running: precompute from the head, which is an earlier block when the last slot was skipped. | +| Gate | Skipped while the sync tracker says syncing, and for a head more than a slot behind the wall clock. At most one worker runs; a key already cached is not recomputed. | +| Work | Clone the head state, `process_slots` to the first slot of `E+1`, flush pending writes, `hash_tree_root` (so the tree nodes' hashes are memoized), send the state back to the actor. | +| Store | The store's state cache under `CacheKey::CheckpointState { epoch: E+1, root: head_root }`. `fork_choice::checkpoint_state` derives the same value for that checkpoint (the checkpoint block's post-state advanced to the epoch's first slot), so attestation targets for `E+1` hit it too. | +| Consume | `on_block` looks up `CheckpointState { block_epoch, parent_root }` when the parent's slot is before the epoch's first slot and the block is at or after it. A hit clones the entry, advances any remaining skipped slots, and applies the block with `stf::apply_block`; the post-state is identical to the inline path's. | +| Miss | Today's path: clone the parent state and run `stf::state_transition`. A block that arrives while the worker is still running takes this path too; the late result is stored and goes unused by it. | + +The code lives in `crates/blockchain/src/epoch_precompute.rs` (triggers, worker, +actor handlers) and `fork_choice::advance_to_epoch_start` / +`fork_choice::transition_block` in the state-transition crate. The upgrade to a +new fork at the boundary happens inside `process_slots`, so the precompute needs +no fork-specific code. Gloas is covered by the same key: a gloas block's +post-state never contains its own payload (the next block applies it inside +`process_block`, after `process_slots`), and the store keeps exactly one state +per root, so the entry for `(E+1, root)` is advanced from the same state an +import of a child of `root` would clone. Hits and misses, worker time and +trigger counts are exported as `lean_beacon_epoch_precompute_*`; see +[metrics](metrics.md). + ## One test per fixture case The suites were once one test apiece, each looping over its own cases and diff --git a/docs/metrics.md b/docs/metrics.md index 64644d7c..529d09a5 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -435,6 +435,45 @@ is a registry scan plus a whole-epoch shuffle on the import thread. `unkeyable` is a lookup the cache could not key at all, mostly the genesis state asking about its own first epochs, and should be zero on a checkpoint-synced follower. +### Beacon Justified Balances + +Beacon fork choice weighs every vote by the voter's effective balance at the +justified checkpoint. `justified_balances` (in +`crates/blockchain/state_transition/src/beacon/fork_choice.rs`) flattens that +state into one array, keyed by the justified checkpoint and rebuilt on the first +`get_head` after the checkpoint moves. This is ethlambda-specific, not part of +the leanMetrics spec. + +| Name | Type | Usage | Sample collection event | Labels | +|------|------|-------|-------------------------|--------| +| `lean_beacon_justified_balances_lookups_total` | Counter | Snapshot lookups, by whether the cached snapshot served them | On every `justified_balances` call: `get_head`'s weights, the proposer boost, and the reorg helpers | result=hit,miss | +| `lean_beacon_justified_balances_build_seconds` | Histogram | Time to build one snapshot from the checkpoint state | On every miss, around the registry pass (the checkpoint state lookup is not included) | | + +**Read the miss rate against the justified-checkpoint rate**: about one miss +per justified checkpoint change, so a handful per hour on a healthy chain. +Misses that track `get_head` calls mean the justified checkpoint is flapping +between branches, or the snapshot is being replaced between two readers. + +### Beacon Epoch Precompute + +The beacon chain actor advances the head state across the next epoch boundary +ahead of time, so the first block of an epoch does not run `process_epoch` on +the import path. See "Epoch-transition precompute" in +[`beacon_stf.md`](beacon_stf.md). This is ethlambda-specific, not part of the +leanMetrics spec. + +| Name | Type | Usage | Sample collection event | Labels | Buckets | +|------|------|-------|-------------------------|--------|---------| +| `lean_beacon_epoch_precompute_lookups_total` | Counter | Epoch-crossing imports, by whether a precomputed boundary state was cached | On each import whose parent is before the block's epoch start and whose block is at or after it | result=hit,miss | | +| `lean_beacon_epoch_precompute_seconds` | Histogram | Worker time to advance, flush and hash one state | When a precompute worker finishes | | 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 4, 8 | +| `lean_beacon_epoch_precompute_started_total` | Counter | Workers started, by trigger | When a worker is spawned | trigger=head,timer | | + +**Read the hit ratio against epochs.** On a synced follower nearly every +epoch-crossing import should hit. Misses mean the worker had not finished (the +block raced it, see `lean_beacon_epoch_precompute_seconds` against the slot +duration), the node was syncing, or the entry was evicted from the state cache. +A `timer` start with no matching `head` start means the last slot was skipped. + ### Beacon Pubkey Cache Every BLS signature check `ethlambda beacon` runs (block import, fork choice,