diff --git a/config.example.toml b/config.example.toml index 1c6354e6..abce5f1d 100644 --- a/config.example.toml +++ b/config.example.toml @@ -184,6 +184,8 @@ loader = "./tests/data/mux_keys.example.json" # loader = { registry = "stader", node_operator_id = 8, stader_pool = "permissionless", enable_refreshing = false } late_in_slot_time_ms = 1500 timeout_get_header_ms = 900 +# (unreleased, from v0.12.0-rc1) ePBS bids only +# proposer_deadline_buffer_ms = 100 # For each mux, one or more [[mux.relays]] can be defined, which will be used for the matching validator pubkeys # Only the relays defined here will be used, and the relays defined in the main [[relays]] config will be ignored # The fields specified here are the same as in [[relays]] (headers, get_params, enable_timing_games, target_first_request_ms, frequency_get_header_ms) diff --git a/crates/common/src/config/mux.rs b/crates/common/src/config/mux.rs index 1b1f0c38..8977e260 100644 --- a/crates/common/src/config/mux.rs +++ b/crates/common/src/config/mux.rs @@ -112,9 +112,12 @@ impl PbsMuxes { late_in_slot_time_ms: mux .late_in_slot_time_ms .unwrap_or(default_pbs.late_in_slot_time_ms), + proposer_deadline_buffer_ms: mux + .proposer_deadline_buffer_ms + .unwrap_or(default_pbs.proposer_deadline_buffer_ms), ..default_pbs.clone() }; - config.validate(chain).await?; + config.validate(chain).await.map_err(|err| eyre::eyre!("mux {}: {err:#}", mux.id))?; let config = Arc::new(config); // Build the map of pubkeys to mux configs @@ -153,6 +156,7 @@ pub struct MuxConfig { pub loader: Option, pub timeout_get_header_ms: Option, pub late_in_slot_time_ms: Option, + pub proposer_deadline_buffer_ms: Option, } impl MuxConfig { diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index f4b57bd7..9ee0e2f3 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -141,7 +141,12 @@ pub async fn get_execution_payload_bid( // No bid could reach the beacon node before its deadline if max_timeout_ms == 0 { - warn!(budget_ms, "proposer deadline reached, no time to solicit a bid"); + warn!( + budget_ms, + buffer_ms = pbs_config.proposer_deadline_buffer_ms, + mux_id = maybe_mux_id.unwrap_or("none"), + "proposer deadline reached, no time to solicit a bid" + ); return Ok(None); } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 0f04c4f9..9096ad08 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -39,7 +39,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i ### 1. Add the builders to Commit-Boost -List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. +List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. A `[[mux]]` can override it for its keys. ### 2. Point each validator key at Commit-Boost {#validator-builder-config} @@ -110,5 +110,5 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | -| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | +| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` in `[pbs]` or a `[[mux]]` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default (`50`; in a `[[mux]]`, the `[pbs]` value) | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/mux-key-loaders.md b/docs/docs/get_started/mux-key-loaders.md index adb72fee..bd3ba4ee 100644 --- a/docs/docs/get_started/mux-key-loaders.md +++ b/docs/docs/get_started/mux-key-loaders.md @@ -6,9 +6,9 @@ description: Mux (multiplexer) configuration and key loader types The PBS multiplexer (or *mux*) lets you route different validators to different relay sets or timing game configurations. Instead of a single `[[relays]]` list for all your validators, you declare one or more `[[mux]]` entries that match specific validator pubkeys to custom relay and timing settings. -A mux covers cases like a Lido or SSV node operator who sends some validators to an operator-specific relay while the rest use the global relay set, or per-group timing games: `timeout_get_header_ms` and `late_in_slot_time_ms` can be set per-mux, overriding the PBS defaults for those validators. The mux key loaders (File, URL, Registry) populate a mux's validator set from a file, an HTTP endpoint, or an on-chain registry, so you don't have to list hundreds or thousands of pubkeys by hand. +A mux covers cases like a Lido or SSV node operator who sends some validators to an operator-specific relay while the rest use the global relay set, or per-group timing games: `timeout_get_header_ms`, `late_in_slot_time_ms` and (unreleased, from v0.12.0-rc1) the ePBS `proposer_deadline_buffer_ms` can be set per-mux, overriding the PBS defaults for those validators. The mux key loaders (File, URL, Registry) populate a mux's validator set from a file, an HTTP endpoint, or an on-chain registry, so you don't have to list hundreds or thousands of pubkeys by hand. -Mux entries are an optional addition to the `[[relays]]` section. A mux affects only `get_header` requests: validator registrations and `submit_blinded_block` always go to all configured relays, global and mux alike. +Mux entries are an optional addition to the `[[relays]]` section. A mux affects `get_header` requests and, for ePBS, bid and builder preferences requests: validator registrations, `submit_blinded_block` and the ePBS signed block always go to all configured relays, global and mux alike. --- diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 09412d2a..b112f05c 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -10,8 +10,8 @@ use alloy::primitives::{B256, U256}; use cb_common::{ config::{ COMMIT_BOOST_IMAGE_DEFAULT, CommitBoostConfig, GetHeaderTransport, HeaderSource, - LogsSettings, ModuleKind, ModuleSigningConfig, PbsConfig, PbsModuleConfig, RelayConfig, - ReverseProxyHeaderSetup, RuntimeMuxConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, + LogsSettings, ModuleKind, ModuleSigningConfig, MuxConfig, PbsConfig, PbsModuleConfig, + PbsMuxes, RelayConfig, ReverseProxyHeaderSetup, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode, }, @@ -344,9 +344,6 @@ pub async fn setup_relays_on_hosts( Ok((setup_pbs(chain, relays, |_| {}).await?, arc_states)) } -/// Poll /status until PBS and its relays are up. relay_check makes a 200 mean -/// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under -/// parallel suite load. /// A PBS with a `[[relays]]` relay and a relay only in the `[[mux]]` of /// `muxed_key`, both on host 0.0.0.0 pub struct MuxedRelays { @@ -358,7 +355,10 @@ pub struct MuxedRelays { pub mux_port: u16, } -pub async fn setup_mux_relays(chain: Chain) -> Result { +pub async fn setup_mux_relays( + chain: Chain, + proposer_deadline_buffer_ms: Option, +) -> Result { let (default_state, default_port) = spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; let (mux_state, mux_port) = @@ -368,14 +368,24 @@ pub async fn setup_mux_relays(chain: Chain) -> Result { let pbs_listener = get_free_listener().await; let pbs_port = pbs_listener.local_addr()?.port(); - let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); - config.all_relays = vec![mux_relay.clone(), default_relay]; + let pbs_config = get_pbs_config(pbs_port); let muxed_key = random_secret().public_key(); - config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { - id: "test".to_string(), - config: config.pbs_config.clone(), - relays: vec![mux_relay], - })])); + let muxes = PbsMuxes { + muxes: vec![MuxConfig { + id: "test".to_string(), + relays: vec![(*mux_relay.config).clone()], + validator_pubkeys: vec![muxed_key.clone()], + loader: None, + // get_pbs_config's own pair fails validation; ePBS routes ignore both + timeout_get_header_ms: Some(u64::MAX - 1), + late_in_slot_time_ms: Some(u64::MAX), + proposer_deadline_buffer_ms, + }], + }; + let (mux_lookup, _) = muxes.validate_and_fill(chain, &pbs_config).await?; + let mut config = to_pbs_config(chain, pbs_config, vec![default_relay.clone()]); + config.all_relays = vec![mux_relay, default_relay]; + config.mux_lookup = Some(mux_lookup); let state = PbsState::new(config, PathBuf::new()); tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); let validator = MockValidator::new(pbs_port)?; @@ -383,6 +393,9 @@ pub async fn setup_mux_relays(chain: Chain) -> Result { Ok(MuxedRelays { validator, muxed_key, default_state, default_port, mux_state, mux_port }) } +/// Poll /status until PBS and its relays are up. relay_check makes a 200 mean +/// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under +/// parallel suite load. pub async fn wait_for_ready(mock_validator: &MockValidator) -> Result<()> { for _ in 0..100 { if let Ok(res) = mock_validator.do_get_status().await && diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 79d04f7e..f968ee7e 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -193,7 +193,7 @@ async fn test_get_execution_payload_bid_dial() -> Result<()> { /// relay only in a mux is not reached so by other keys: it is dialed instead #[tokio::test] async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> { - let relays = setup_mux_relays(Chain::Hoodi).await?; + let relays = setup_mux_relays(Chain::Hoodi, None).await?; let bid = async |key: Option, auth_data: &[u8]| { relays .validator @@ -228,6 +228,40 @@ async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> Ok(()) } +/// A key in a mux gets the mux's `proposer_deadline_buffer_ms`, other keys +/// `[pbs]`'s, and a mux's `0` does not load +#[tokio::test] +async fn test_get_execution_payload_bid_mux_buffer() -> Result<()> { + let relays = setup_mux_relays(Chain::Hoodi, Some(2_000)).await?; + let bid = async |key: Option| { + relays + .validator + .do_get_execution_payload_bid_with_timeout( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + key, + Some(&opaque_auth(TEST_AUTH_DATA, TEST_SLOT)), + vec![EncodingType::Json], + 60_000, + ) + .await + }; + assert_eq!(bid(Some(relays.muxed_key.clone())).await?.status(), StatusCode::OK); + assert_eq!(bid(None).await?.status(), StatusCode::OK); + + // One 12 s slot, less each buffer + assert_eq!(relays.mux_state.received_bid_timeout_ms(), Some(10_000)); + assert_eq!(relays.default_state.received_bid_timeout_ms(), Some(11_950)); + + let err = setup_mux_relays(Chain::Hoodi, Some(0)).await.err().expect("a buffer of 0 loaded"); + assert!( + err.to_string().contains("proposer_deadline_buffer_ms must be greater than 0"), + "{err}" + ); + Ok(()) +} + /// `auth.message.slot` must match the proposal slot in the request path. #[tokio::test] async fn test_get_execution_payload_bid_auth_slot_mismatch_400() -> Result<()> { diff --git a/tests/tests/pbs_mux.rs b/tests/tests/pbs_mux.rs index 6f01f502..4014b833 100644 --- a/tests/tests/pbs_mux.rs +++ b/tests/tests/pbs_mux.rs @@ -383,6 +383,7 @@ async fn test_ssv_multi_with_node() -> Result<()> { late_in_slot_time_ms: Some(u64::MAX), relays: vec![(*relay.config).clone()], timeout_get_header_ms: Some(u64::MAX - 1), + proposer_deadline_buffer_ms: None, validator_pubkeys: vec![], }], }; @@ -490,6 +491,7 @@ async fn test_ssv_multi_with_public() -> Result<()> { late_in_slot_time_ms: Some(u64::MAX), relays: vec![(*relay.config).clone()], timeout_get_header_ms: Some(u64::MAX - 1), + proposer_deadline_buffer_ms: None, validator_pubkeys: vec![], }], }; diff --git a/tests/tests/pbs_mux_refresh.rs b/tests/tests/pbs_mux_refresh.rs index 10a9f1c8..b3a81e5b 100644 --- a/tests/tests/pbs_mux_refresh.rs +++ b/tests/tests/pbs_mux_refresh.rs @@ -92,6 +92,7 @@ async fn test_auto_refresh() -> Result<()> { late_in_slot_time_ms: Some(u64::MAX), relays: vec![(*mux_relay.config).clone()], timeout_get_header_ms: Some(u64::MAX - 1), + proposer_deadline_buffer_ms: None, validator_pubkeys: vec![], }], }; diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs index 53a5208d..e70d09a7 100644 --- a/tests/tests/pbs_submit_builder_preferences.rs +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -294,7 +294,7 @@ async fn test_submit_builder_preferences_dial() -> Result<()> { /// by other keys: it is dialed, and the dial check refuses 0.0.0.0 #[tokio::test] async fn test_submit_builder_preferences_mux_key_reaches_a_relays_entry() -> Result<()> { - let relays = setup_mux_relays(Chain::Hoodi).await?; + let relays = setup_mux_relays(Chain::Hoodi, None).await?; let mock_validator = &relays.validator; let (muxed_key, default_port, mux_port) = (relays.muxed_key.clone(), relays.default_port, relays.mux_port);