feat(pbs): let a mux override proposer_deadline_buffer_ms

config.example.toml

@@ -184,6 +184,8 @@ loader = "./tests/data/mux_keys.example.json"
 # loader = { registry = "stader", node_operator_id = 8, stader_pool = "permissionless", enable_refreshing = false }
 late_in_slot_time_ms = 1500
 timeout_get_header_ms = 900
+# (unreleased, from v0.12.0-rc1) ePBS bids only
+# proposer_deadline_buffer_ms = 100
 # For each mux, one or more [[mux.relays]] can be defined, which will be used for the matching validator pubkeys
 # Only the relays defined here will be used, and the relays defined in the main [[relays]] config will be ignored
 # The fields specified here are the same as in [[relays]] (headers, get_params, enable_timing_games, target_first_request_ms, frequency_get_header_ms)

crates/common/src/config/mux.rs

@@ -112,9 +112,12 @@ impl PbsMuxes {
                 late_in_slot_time_ms: mux
                     .late_in_slot_time_ms
                     .unwrap_or(default_pbs.late_in_slot_time_ms),
+                proposer_deadline_buffer_ms: mux
+                    .proposer_deadline_buffer_ms
+                    .unwrap_or(default_pbs.proposer_deadline_buffer_ms),
                 ..default_pbs.clone()
             };
-            config.validate(chain).await?;
+            config.validate(chain).await.map_err(|err| eyre::eyre!("mux {}: {err:#}", mux.id))?;
             let config = Arc::new(config);
 
             // Build the map of pubkeys to mux configs
@@ -153,6 +156,7 @@ pub struct MuxConfig {
     pub loader: Option<MuxKeysLoader>,
     pub timeout_get_header_ms: Option<u64>,
     pub late_in_slot_time_ms: Option<u64>,
+    pub proposer_deadline_buffer_ms: Option<u64>,
 }
 
 impl MuxConfig {

crates/pbs/src/routes/execution_payload_bid.rs

@@ -141,7 +141,12 @@ pub async fn get_execution_payload_bid<S: BuilderApiState>(
 
     // No bid could reach the beacon node before its deadline
     if max_timeout_ms == 0 {
-        warn!(budget_ms, "proposer deadline reached, no time to solicit a bid");
+        warn!(
+            budget_ms,
+            buffer_ms = pbs_config.proposer_deadline_buffer_ms,
+            mux_id = maybe_mux_id.unwrap_or("none"),
+            "proposer deadline reached, no time to solicit a bid"
+        );
         return Ok(None);
     }
 

docs/docs/get_started/epbs.md

@@ -39,7 +39,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i
 
 ### 1. Add the builders to Commit-Boost
 
-List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet.
+List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. A `[[mux]]` can override it for its keys.
 
 ### 2. Point each validator key at Commit-Boost {#validator-builder-config}
 
@@ -110,5 +110,5 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi
 | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches |
 | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) |
 | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them |
-| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` |
+| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` in `[pbs]` or a `[[mux]]` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default (`50`; in a `[[mux]]`, the `[pbs]` value) |
 | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` |

docs/docs/get_started/mux-key-loaders.md

@@ -6,9 +6,9 @@ description: Mux (multiplexer) configuration and key loader types
 
 The PBS multiplexer (or *mux*) lets you route different validators to different relay sets or timing game configurations. Instead of a single `[[relays]]` list for all your validators, you declare one or more `[[mux]]` entries that match specific validator pubkeys to custom relay and timing settings.
 
-A mux covers cases like a Lido or SSV node operator who sends some validators to an operator-specific relay while the rest use the global relay set, or per-group timing games: `timeout_get_header_ms` and `late_in_slot_time_ms` can be set per-mux, overriding the PBS defaults for those validators. The mux key loaders (File, URL, Registry) populate a mux's validator set from a file, an HTTP endpoint, or an on-chain registry, so you don't have to list hundreds or thousands of pubkeys by hand.
+A mux covers cases like a Lido or SSV node operator who sends some validators to an operator-specific relay while the rest use the global relay set, or per-group timing games: `timeout_get_header_ms`, `late_in_slot_time_ms` and (unreleased, from v0.12.0-rc1) the ePBS `proposer_deadline_buffer_ms` can be set per-mux, overriding the PBS defaults for those validators. The mux key loaders (File, URL, Registry) populate a mux's validator set from a file, an HTTP endpoint, or an on-chain registry, so you don't have to list hundreds or thousands of pubkeys by hand.
 
-Mux entries are an optional addition to the `[[relays]]` section. A mux affects only `get_header` requests: validator registrations and `submit_blinded_block` always go to all configured relays, global and mux alike.
+Mux entries are an optional addition to the `[[relays]]` section. A mux affects `get_header` requests and, for ePBS, bid and builder preferences requests: validator registrations, `submit_blinded_block` and the ePBS signed block always go to all configured relays, global and mux alike.
 
 ---
 

tests/src/utils.rs

@@ -10,8 +10,8 @@ use alloy::primitives::{B256, U256};
 use cb_common::{
     config::{
         COMMIT_BOOST_IMAGE_DEFAULT, CommitBoostConfig, GetHeaderTransport, HeaderSource,
-        LogsSettings, ModuleKind, ModuleSigningConfig, PbsConfig, PbsModuleConfig, RelayConfig,
-        ReverseProxyHeaderSetup, RuntimeMuxConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT,
+        LogsSettings, ModuleKind, ModuleSigningConfig, MuxConfig, PbsConfig, PbsModuleConfig,
+        PbsMuxes, RelayConfig, ReverseProxyHeaderSetup, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT,
         SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig,
         SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode,
     },
@@ -344,9 +344,6 @@ pub async fn setup_relays_on_hosts(
     Ok((setup_pbs(chain, relays, |_| {}).await?, arc_states))
 }
 
-/// Poll /status until PBS and its relays are up. relay_check makes a 200 mean
-/// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under
-/// parallel suite load.
 /// A PBS with a `[[relays]]` relay and a relay only in the `[[mux]]` of
 /// `muxed_key`, both on host 0.0.0.0
 pub struct MuxedRelays {
@@ -358,7 +355,10 @@ pub struct MuxedRelays {
     pub mux_port: u16,
 }
 
-pub async fn setup_mux_relays(chain: Chain) -> Result<MuxedRelays> {
+pub async fn setup_mux_relays(
+    chain: Chain,
+    proposer_deadline_buffer_ms: Option<u64>,
+) -> Result<MuxedRelays> {
     let (default_state, default_port) =
         spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?;
     let (mux_state, mux_port) =
@@ -368,21 +368,34 @@ pub async fn setup_mux_relays(chain: Chain) -> Result<MuxedRelays> {
 
     let pbs_listener = get_free_listener().await;
     let pbs_port = pbs_listener.local_addr()?.port();
-    let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]);
-    config.all_relays = vec![mux_relay.clone(), default_relay];
+    let pbs_config = get_pbs_config(pbs_port);
     let muxed_key = random_secret().public_key();
-    config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig {
-        id: "test".to_string(),
-        config: config.pbs_config.clone(),
-        relays: vec![mux_relay],
-    })]));
+    let muxes = PbsMuxes {
+        muxes: vec![MuxConfig {
+            id: "test".to_string(),
+            relays: vec![(*mux_relay.config).clone()],
+            validator_pubkeys: vec![muxed_key.clone()],
+            loader: None,
+            // get_pbs_config's own pair fails validation; ePBS routes ignore both
+            timeout_get_header_ms: Some(u64::MAX - 1),
+            late_in_slot_time_ms: Some(u64::MAX),
+            proposer_deadline_buffer_ms,
+        }],
+    };
+    let (mux_lookup, _) = muxes.validate_and_fill(chain, &pbs_config).await?;
+    let mut config = to_pbs_config(chain, pbs_config, vec![default_relay.clone()]);
+    config.all_relays = vec![mux_relay, default_relay];
+    config.mux_lookup = Some(mux_lookup);
     let state = PbsState::new(config, PathBuf::new());
     tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener));
     let validator = MockValidator::new(pbs_port)?;
     wait_for_ready(&validator).await?;
     Ok(MuxedRelays { validator, muxed_key, default_state, default_port, mux_state, mux_port })
 }
 
+/// Poll /status until PBS and its relays are up. relay_check makes a 200 mean
+/// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under
+/// parallel suite load.
 pub async fn wait_for_ready(mock_validator: &MockValidator) -> Result<()> {
     for _ in 0..100 {
         if let Ok(res) = mock_validator.do_get_status().await &&

tests/tests/pbs_get_execution_payload_bid.rs

@@ -193,7 +193,7 @@ async fn test_get_execution_payload_bid_dial() -> Result<()> {
 /// relay only in a mux is not reached so by other keys: it is dialed instead
 #[tokio::test]
 async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> {
-    let relays = setup_mux_relays(Chain::Hoodi).await?;
+    let relays = setup_mux_relays(Chain::Hoodi, None).await?;
     let bid = async |key: Option<BlsPublicKey>, auth_data: &[u8]| {
         relays
             .validator
@@ -228,6 +228,40 @@ async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()>
     Ok(())
 }
 
+/// A key in a mux gets the mux's `proposer_deadline_buffer_ms`, other keys
+/// `[pbs]`'s, and a mux's `0` does not load
+#[tokio::test]
+async fn test_get_execution_payload_bid_mux_buffer() -> Result<()> {
+    let relays = setup_mux_relays(Chain::Hoodi, Some(2_000)).await?;
+    let bid = async |key: Option<BlsPublicKey>| {
+        relays
+            .validator
+            .do_get_execution_payload_bid_with_timeout(
+                TEST_SLOT,
+                B256::ZERO,
+                B256::ZERO,
+                key,
+                Some(&opaque_auth(TEST_AUTH_DATA, TEST_SLOT)),
+                vec![EncodingType::Json],
+                60_000,
+            )
+            .await
+    };
+    assert_eq!(bid(Some(relays.muxed_key.clone())).await?.status(), StatusCode::OK);
+    assert_eq!(bid(None).await?.status(), StatusCode::OK);
+
+    // One 12 s slot, less each buffer
+    assert_eq!(relays.mux_state.received_bid_timeout_ms(), Some(10_000));
+    assert_eq!(relays.default_state.received_bid_timeout_ms(), Some(11_950));
+
+    let err = setup_mux_relays(Chain::Hoodi, Some(0)).await.err().expect("a buffer of 0 loaded");
+    assert!(
+        err.to_string().contains("proposer_deadline_buffer_ms must be greater than 0"),
+        "{err}"
+    );
+    Ok(())
+}
+
 /// `auth.message.slot` must match the proposal slot in the request path.
 #[tokio::test]
 async fn test_get_execution_payload_bid_auth_slot_mismatch_400() -> Result<()> {

tests/tests/pbs_mux.rs

@@ -383,6 +383,7 @@ async fn test_ssv_multi_with_node() -> Result<()> {
             late_in_slot_time_ms: Some(u64::MAX),
             relays: vec![(*relay.config).clone()],
             timeout_get_header_ms: Some(u64::MAX - 1),
+            proposer_deadline_buffer_ms: None,
             validator_pubkeys: vec![],
         }],
     };
@@ -490,6 +491,7 @@ async fn test_ssv_multi_with_public() -> Result<()> {
             late_in_slot_time_ms: Some(u64::MAX),
             relays: vec![(*relay.config).clone()],
             timeout_get_header_ms: Some(u64::MAX - 1),
+            proposer_deadline_buffer_ms: None,
             validator_pubkeys: vec![],
         }],
     };

tests/tests/pbs_mux_refresh.rs

@@ -92,6 +92,7 @@ async fn test_auto_refresh() -> Result<()> {
             late_in_slot_time_ms: Some(u64::MAX),
             relays: vec![(*mux_relay.config).clone()],
             timeout_get_header_ms: Some(u64::MAX - 1),
+            proposer_deadline_buffer_ms: None,
             validator_pubkeys: vec![],
         }],
     };

tests/tests/pbs_submit_builder_preferences.rs

@@ -294,7 +294,7 @@ async fn test_submit_builder_preferences_dial() -> Result<()> {
 /// by other keys: it is dialed, and the dial check refuses 0.0.0.0
 #[tokio::test]
 async fn test_submit_builder_preferences_mux_key_reaches_a_relays_entry() -> Result<()> {
-    let relays = setup_mux_relays(Chain::Hoodi).await?;
+    let relays = setup_mux_relays(Chain::Hoodi, None).await?;
     let mock_validator = &relays.validator;
     let (muxed_key, default_port, mux_port) =
         (relays.muxed_key.clone(), relays.default_port, relays.mux_port);