diff --git a/crates/metadb-cluster/src/cell_manager.rs b/crates/metadb-cluster/src/cell_manager.rs index f7d8f86..4275881 100644 --- a/crates/metadb-cluster/src/cell_manager.rs +++ b/crates/metadb-cluster/src/cell_manager.rs @@ -980,6 +980,25 @@ impl DurableCellManager { Ok(()) } + /// Routes for cluster status. A local shard that cannot rebuild its snapshot + /// is not returned as a live voter; the caller gets an error instead. + pub fn routes_for_live_status(&self) -> Result> { + let routes = self.list_routes(); + for route in &routes { + if let Some(group) = self.host.get_group(route.shard_id) { + // Probe the snapshot directory. The in-memory latch is still + // empty until something attempts repair. + group.ensure_published_snapshot_sync().map_err(|error| { + MetaDbError::StorageIoError(format!( + "shard {} storage is unrecoverable, so it is not a live voter: {error}", + route.shard_id + )) + })?; + } + } + Ok(routes) + } + pub fn list_routes(&self) -> Vec { let mut routes: Vec = { let guard = self.state.lock().unwrap(); diff --git a/crates/metadb-raft/src/group_host.rs b/crates/metadb-raft/src/group_host.rs index b3dd036..e2456aa 100644 --- a/crates/metadb-raft/src/group_host.rs +++ b/crates/metadb-raft/src/group_host.rs @@ -10,6 +10,7 @@ use std::time::Duration; use openraft::async_runtime::watch::WatchReceiver; use openraft::errors::RaftError; +use openraft::storage::RaftStateMachine; use openraft::{BasicNode, Config}; use serde::{Deserialize, Serialize}; @@ -241,6 +242,12 @@ impl ShardRaftGroup { let _ = raft.wait_for_recovery(Some(Duration::from_secs(5))).await; } + if let Some(reason) = state_machine.storage_unrecoverable() { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: {reason}" + ))); + } + router.register_group(shard_id, raft.clone()); let read_batcher = Arc::new(ReadIndexBatcher::new(shard_id, raft.clone(), frontier_rx)); @@ -389,6 +396,57 @@ impl ShardRaftGroup { &self.state_machine } + /// `Some` when this process must not be reported as a live voter. + pub fn storage_unrecoverable(&self) -> Option { + self.state_machine.storage_unrecoverable() + } + + /// Repair a missing published snapshot before cluster status returns. + /// + /// Status must not trust the in-memory latch alone. The files can disappear + /// before any `get_current_snapshot` call. + pub fn ensure_published_snapshot_sync(&self) -> Result<()> { + if let Some(reason) = self.storage_unrecoverable() { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: {reason}" + ))); + } + if !self.state_machine.published_snapshot_missing() { + return Ok(()); + } + self.state_machine + .ensure_durable_snapshot() + .map_err(|error| MetaDbError::StorageIoError(error.to_string()))?; + if let Some(reason) = self.storage_unrecoverable() { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: {reason}" + ))); + } + Ok(()) + } + + /// Rewrite a published snapshot that disappeared, or refuse to serve. + /// + /// A missing in-use snapshot used to stop RaftCore while cluster status + /// still answered from the cached voter set. Repair from the LSM when it + /// can cover the purged log. Otherwise leave an explicit unrecoverable + /// error so status and later reads do not pretend the voter is healthy. + pub async fn repair_published_snapshot(&self) -> Result<()> { + if let Some(reason) = self.storage_unrecoverable() { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: {reason}" + ))); + } + if !self.state_machine.published_snapshot_missing() { + return Ok(()); + } + let mut state_machine = self.state_machine.clone(); + match state_machine.get_current_snapshot().await { + Ok(_) => Ok(()), + Err(error) => Err(MetaDbError::StorageIoError(error.to_string())), + } + } + pub fn engine(&self) -> Arc { self.state_machine.engine() } @@ -647,6 +705,8 @@ impl ShardRaftGroup { caller_hlc: HlcTimestamp, payload: Vec, ) -> Result { + // Forwarded ClientWrite RPCs enter here and skip `propose_until`. + self.repair_published_snapshot().await?; let local_epoch = self.current_epoch(); if expected_epoch != local_epoch { return Err(MetaDbError::EpochMismatch { @@ -750,6 +810,7 @@ impl ShardRaftGroup { &self, expected_epoch: ShardEpoch, ) -> Result<(ReadToken, SnapshotGuard)> { + self.repair_published_snapshot().await?; if self .testing_read_quorum .load(std::sync::atomic::Ordering::Relaxed) @@ -778,6 +839,7 @@ impl ShardRaftGroup { expected_epoch: ShardEpoch, barrier_hlc: HlcTimestamp, ) -> Result<(ReadToken, SnapshotGuard)> { + self.repair_published_snapshot().await?; let _ = self.clock.update_checked(barrier_hlc)?; let token = self.read_batcher.linearizable_token(expected_epoch).await?; let snap = self.state_machine.acquire_snapshot( diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index 52306b6..6313c2f 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -6,6 +6,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, RwLock}; +use std::time::SystemTime; use openraft::entry::RaftEntry; use openraft::storage::{EntryResponder, RaftSnapshotBuilder, RaftStateMachine}; @@ -240,6 +241,18 @@ struct StateMachineShared { last_applied_hlc: RwLock, last_membership: RwLock>, frontier_tx: watch::Sender, + /// Set when a required snapshot cannot be rebuilt. Cluster status must not + /// keep reporting this process as a live voter after RaftCore has failed. + unrecoverable: Mutex>, + /// Last `SNAPSHOT_META` stat that passed a full completeness check. + /// Later requests compare this stat instead of decoding every SSTable. + snapshot_cache: Mutex>, +} + +struct PublishedSnapshotCache { + index: u64, + meta_len: u64, + meta_modified: Option, } impl StateMachineShared { @@ -415,7 +428,20 @@ impl StateMachineShared { // Persist SNAPSHOT_META's directory entry before publishing the snapshot. sync_dir(&tmp_dir).map_err(|e| io::Error::other(e.to_string()))?; + let purged = crate::log_store::read_persisted_purged_index(&self.shard_dir.join("raft")) + .unwrap_or(0); + let required = purged.max(self.watermarks.snapshot_values().2); + if snap_index < required { + let _ = fs::remove_dir_all(&tmp_dir); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "refusing to publish snapshot index {snap_index}; it does not cover required index {required}" + ), + )); + } publish_snapshot_dir(&snaps_root, snap_index, &tmp_dir)?; + *self.snapshot_cache.lock().unwrap() = None; self.watermarks .advance_last_snapshot(checkpoint.flushed_raft_index); @@ -426,23 +452,114 @@ impl StateMachineShared { }) } + fn clear_unrecoverable(&self) { + *self.unrecoverable.lock().unwrap() = None; + } + + fn fail_unrecoverable(&self, reason: impl Into) -> io::Result { + let reason = reason.into(); + *self.unrecoverable.lock().unwrap() = Some(reason.clone()); + Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("snapshot unrecoverable: {reason}"), + )) + } + + /// True when the published snapshot directory is missing or no longer matches + /// the `SNAPSHOT_META` stat that last passed a full check. + /// + /// The common path is one metadata call. A changed or new meta file is + /// validated once and then cached, so a healthy request does not decode + /// every SSTable. + fn published_snapshot_missing(&self) -> bool { + let index = self.watermarks.snapshot_values().2; + if index == 0 { + return false; + } + let meta_path = self + .shard_dir + .join(format!("raft/snapshots/snap_{index:020}/SNAPSHOT_META")); + let Ok(info) = fs::metadata(&meta_path) else { + *self.snapshot_cache.lock().unwrap() = None; + return true; + }; + if !info.is_file() || info.len() <= 4 { + *self.snapshot_cache.lock().unwrap() = None; + return true; + } + let modified = info.modified().ok(); + if self + .snapshot_cache + .lock() + .unwrap() + .as_ref() + .is_some_and(|cached| { + cached.index == index + && cached.meta_len == info.len() + && cached.meta_modified == modified + }) + { + return false; + } + let complete = meta_path.parent().is_some_and(snapshot_dir_is_complete); + if complete { + *self.snapshot_cache.lock().unwrap() = Some(PublishedSnapshotCache { + index, + meta_len: info.len(), + meta_modified: modified, + }); + false + } else { + *self.snapshot_cache.lock().unwrap() = None; + true + } + } + + fn required_snapshot_index(&self) -> u64 { + let purged = crate::log_store::read_persisted_purged_index(&self.shard_dir.join("raft")) + .unwrap_or(0); + purged.max(self.watermarks.snapshot_values().2) + } + fn load_current_snapshot_sync( &self, ) -> io::Result>> { let snaps_root = self.shard_dir.join("raft/snapshots"); + let candidates = snapshot_candidates(&snaps_root); + let required = self.required_snapshot_index(); // A missing file used to return `NotFound`. OpenRaft maps that to // `StorageError { subject: Snapshot(None), verb: Read }` and stops RaftCore, - // after which every later read repeats the same error. Skip a broken - // directory and use the newest complete snapshot instead, including the - // `.bak` copy left by an in-progress replace. - for (_, snap_dir) in snapshot_candidates(&snaps_root) { - match load_snapshot_dir(&snap_dir) { - Ok(snapshot) => return Ok(Some(snapshot)), - Err(error) if error.kind() == io::ErrorKind::NotFound => { - tracing::warn!( - path = %snap_dir.display(), - "skipping raft snapshot directory with a missing file" - ); + // while the process kept answering cluster status from cached membership. + // An older readable directory does not satisfy a higher purged or published index. + for (index, snap_dir) in &candidates { + if *index < required { + tracing::warn!( + path = %snap_dir.display(), + index, + required, + "skipping raft snapshot that does not cover the required index" + ); + continue; + } + match load_snapshot_dir(snap_dir) { + Ok(snapshot) => { + let covered = snapshot + .meta + .last_log_id + .as_ref() + .map(|id| id.index) + .unwrap_or(0); + if covered < required { + tracing::warn!( + path = %snap_dir.display(), + covered, + required, + "skipping raft snapshot whose log id is behind the required index" + ); + continue; + } + self.clear_unrecoverable(); + return Ok(Some(snapshot)); } Err(error) => { tracing::warn!( @@ -453,7 +570,62 @@ impl StateMachineShared { } } } - Ok(None) + + let applied = self + .last_applied_log_id + .read() + .unwrap() + .map(|id| id.index) + .unwrap_or(0); + let flushed = self + .current_engine() + .durable_checkpoint() + .flushed_raft_index; + // The segmented raft log is the WAL until it is purged. Before that, + // a missing snapshot file is not fatal: restart replays the log. + if required == 0 { + self.clear_unrecoverable(); + return Ok(None); + } + + // After purge, the LSM is the only remaining copy. Rewrite the snapshot + // from it before returning, so a live process does not keep serving + // with the snapshot file gone. `build_snapshot_sync` refuses to publish + // a directory that does not cover `required`. + match self.build_snapshot_sync() { + Ok(snapshot) => { + let covered = snapshot + .meta + .last_log_id + .as_ref() + .map(|id| id.index) + .unwrap_or(0); + if required > covered { + return self.fail_unrecoverable(format!( + "rebuilt snapshot index {covered} does not cover required index {required}. The segmented raft log is the WAL only until it is purged. A missing snapshot is rebuilt from the LSM. If that LSM is also missing, committed data is not reconstructed" + )); + } + self.clear_unrecoverable(); + Ok(Some(snapshot)) + } + Err(error) => { + let text = error.to_string(); + // A transient publish error must not stick. Latch only when the + // flushed LSM cannot cover the purged prefix, or the files + // needed to rebuild it are already gone. + let files_gone = text.contains("os error 2") + || text.contains("No such file") + || text.contains("does not cover required") + || text.contains("not found"); + if flushed < required || files_gone { + self.fail_unrecoverable(format!( + "cannot rebuild a missing snapshot from the LSM ({text}). required index is {required}, flushed LSM index is {flushed}, applied index is {applied}. After purge, reconstruction depends on the LSM. There is no fourth copy, so committed data is not reconstructed" + )) + } else { + Err(error) + } + } + } } } @@ -712,12 +884,24 @@ impl ShardStateMachine { } } - let engine = Arc::new(LifecycleStorageEngine::open_lsm( + let engine = match LifecycleStorageEngine::open_lsm( shard_id, &lsm_dir, block_cache.clone(), lsm_config.clone(), - )?); + ) { + Ok(engine) => Arc::new(engine), + Err(error) => { + let purged = crate::log_store::read_persisted_purged_index(&shard_dir.join("raft")) + .unwrap_or(0); + if purged > 0 { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: cannot open the LSM after the raft log was purged ({error}). committed data is not reconstructed" + ))); + } + return Err(error); + } + }; attach_flush_hook(&engine, &handler); let cp = engine.durable_checkpoint(); @@ -759,8 +943,21 @@ impl ShardStateMachine { last_applied_hlc: RwLock::new(cp.flushed_max_hlc), last_membership: RwLock::new(initial_membership), frontier_tx, + unrecoverable: Mutex::new(None), + snapshot_cache: Mutex::new(None), }); + // Rewrite a missing snapshot, or refuse to return a state machine that + // would serve an empty LSM over a purged log. + if let Err(error) = shared.load_current_snapshot_sync() { + return Err(MetaDbError::StorageIoError(error.to_string())); + } + if let Some(reason) = shared.unrecoverable.lock().unwrap().clone() { + return Err(MetaDbError::StorageIoError(format!( + "snapshot unrecoverable: {reason}" + ))); + } + Ok((Self { shared }, frontier_rx)) } @@ -768,6 +965,21 @@ impl ShardStateMachine { self.shared.current_engine() } + /// Reason this process must not be advertised as a live voter. + pub fn storage_unrecoverable(&self) -> Option { + self.shared.unrecoverable.lock().unwrap().clone() + } + + /// The snapshot index already published in memory has no complete directory. + pub fn published_snapshot_missing(&self) -> bool { + self.shared.published_snapshot_missing() + } + + /// Rebuild a required snapshot from the LSM, or record that it cannot be covered. + pub fn ensure_durable_snapshot(&self) -> io::Result<()> { + self.shared.load_current_snapshot_sync().map(|_| ()) + } + pub fn handler(&self) -> &Arc { &self.shared.handler } diff --git a/crates/metadb-raft/tests/snapshot_read_path_tests.rs b/crates/metadb-raft/tests/snapshot_read_path_tests.rs index 853a69b..380600f 100644 --- a/crates/metadb-raft/tests/snapshot_read_path_tests.rs +++ b/crates/metadb-raft/tests/snapshot_read_path_tests.rs @@ -1,14 +1,20 @@ //! A purged or missing Raft snapshot file must not stop the read path. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::sync::Arc; +use std::time::Duration; + +use tokio::net::TcpListener; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::transport::Server; use metadb_core::codec; use metadb_core::{HlcTimestamp, HybridLogicalClock, MetaKey, ShardEpoch}; +use metadb_proto::raft_transport_service_server::RaftTransportServiceServer; use metadb_raft::{ - BasicKvApplyHandler, BasicKvCommand, ChannelPool, MultiRaftRouter, RaftNodeOptions, - ShardRaftGroup, ShardWatermarkTracker, + BasicKvApplyHandler, BasicKvCommand, ChannelPool, ConfChangeOrchestrator, MultiRaftRouter, + MultiRaftTransportServer, RaftNodeOptions, ShardRaftGroup, ShardWatermarkTracker, }; use metadb_storage::{LsmEngineConfig, ShardedBlockCache}; use openraft::storage::RaftStateMachine; @@ -116,8 +122,27 @@ async fn missing_snapshot_after_flush_and_purge_does_not_kill_reads() { let mut state_machine = group.state_machine().clone(); let loaded = state_machine.get_current_snapshot().await.unwrap(); assert!( - loaded.is_none(), - "a missing snapshot file must not be returned as a fatal read error" + loaded.is_some(), + "a live process must rewrite the missing snapshot from the LSM" + ); + assert!( + group.storage_unrecoverable().is_none(), + "a rebuilt snapshot must still be reported as a live voter" + ); + assert!( + shard_dir + .join("raft/snapshots") + .read_dir() + .unwrap() + .any(|entry| { + entry + .unwrap() + .file_name() + .to_str() + .unwrap_or("") + .starts_with("snap_") + }), + "rewrite must put a snapshot directory back on disk" ); assert_keys(&group).await; group.shutdown().await.unwrap(); @@ -133,3 +158,479 @@ async fn missing_snapshot_after_flush_and_purge_does_not_kill_reads() { assert_keys(&group).await; group.shutdown().await.unwrap(); } + +#[tokio::test] +async fn unpurged_raft_log_reconstructs_keys_without_a_snapshot() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + { + let group = boot(&shard_dir, cache.clone(), clock.clone()).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + assert!( + group.log_store().retained_entry_count() > 0, + "the segmented log is the WAL until purge" + ); + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + group.shutdown().await.unwrap(); + } + let group = boot(&shard_dir, cache, clock).await; + assert!( + group.storage_unrecoverable().is_none(), + "an unpurged log must not be reported as an unrecoverable voter" + ); + assert_keys(&group).await; + group.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn missing_lsm_and_snapshot_after_log_purge_is_not_a_live_voter() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + let group = boot(&shard_dir, cache, clock).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + let purged = group.flush_snapshot_and_purge().await.unwrap(); + assert!(purged > 0); + assert_eq!(group.log_store().retained_entry_count(), 0); + + let mut removed_sst = 0usize; + for entry in fs::read_dir(shard_dir.join("lsm")).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if name.ends_with(".sst") { + fs::remove_file(entry.path()).unwrap(); + removed_sst += 1; + } + } + assert!(removed_sst > 0, "flush did not create an SSTable to remove"); + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + + let mut state_machine = group.state_machine().clone(); + let error = state_machine + .get_current_snapshot() + .await + .expect_err("missing LSM and snapshot must not look like a healthy snapshot"); + let message = error.to_string(); + assert!( + message.contains("snapshot unrecoverable"), + "missing durable copies must be an explicit error, not ENOENT: {message}" + ); + assert!( + message.contains("not reconstructed"), + "the error must say committed data is not reconstructed: {message}" + ); + let reason = group + .storage_unrecoverable() + .expect("cluster status must be able to see that this process is not a live voter"); + assert!(reason.contains("not reconstructed"), "{reason}"); + let read_error = match group.linearizable_read(group.current_epoch()).await { + Ok(_) => panic!("reads must not succeed once storage is unrecoverable"), + Err(error) => error, + }; + assert!( + read_error.to_string().contains("snapshot unrecoverable"), + "{read_error}" + ); + group.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn forwarded_propose_local_rewrites_a_missing_snapshot() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + let group = boot(&shard_dir, cache, clock).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + group.flush_snapshot_and_purge().await.unwrap(); + let published = group.watermarks().snapshot_values().2; + let meta = shard_dir.join(format!("raft/snapshots/snap_{published:020}/SNAPSHOT_META")); + fs::remove_file(&meta).unwrap(); + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, "delta").encode(), + value: b"delta".to_vec(), + }; + group + .propose_local(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + let rewritten = shard_dir + .join("raft/snapshots") + .read_dir() + .unwrap() + .any(|entry| { + entry + .ok() + .and_then(|entry| entry.file_name().into_string().ok()) + .is_some_and(|name| name.starts_with("snap_") && !name.contains('.')) + }); + assert!(rewritten, "propose_local must rewrite the missing snapshot"); + assert!(group.storage_unrecoverable().is_none()); + group.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn older_snapshot_does_not_satisfy_a_purged_prefix() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + let group = boot(&shard_dir, cache, clock).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + let purged = group.flush_snapshot_and_purge().await.unwrap(); + assert!(purged > 1); + for entry in fs::read_dir(shard_dir.join("lsm")).unwrap() { + let entry = entry.unwrap(); + if entry.file_name().to_str().unwrap_or("").ends_with(".sst") { + fs::remove_file(entry.path()).unwrap(); + } + } + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + metadb_raft::state_machine::write_complete_snapshot_marker(&shard_dir.join("raft"), 1).unwrap(); + + let mut state_machine = group.state_machine().clone(); + let first = state_machine.get_current_snapshot().await.unwrap_err(); + assert!( + first.to_string().contains("snapshot unrecoverable"), + "{first}" + ); + let second = state_machine.get_current_snapshot().await.unwrap_err(); + assert!( + second.to_string().contains("snapshot unrecoverable"), + "a later read must not accept the older snapshot: {second}" + ); + assert!(group.storage_unrecoverable().is_some()); + group.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn restart_after_purged_log_missing_lsm_and_snapshot_is_unrecoverable_before_serving() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + { + let group = boot(&shard_dir, cache.clone(), clock.clone()).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + let purged = group.flush_snapshot_and_purge().await.unwrap(); + assert!(purged > 0); + for entry in fs::read_dir(shard_dir.join("lsm")).unwrap() { + let entry = entry.unwrap(); + if entry.file_name().to_str().unwrap_or("").ends_with(".sst") { + fs::remove_file(entry.path()).unwrap(); + } + } + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + group.shutdown().await.unwrap(); + } + + let watermarks = Arc::new(ShardWatermarkTracker::new(0, 0, 0)); + let handler = Arc::new(BasicKvApplyHandler::new( + ShardEpoch::initial(), + watermarks.clone(), + )); + let router = MultiRaftRouter::new(); + let started = ShardRaftGroup::start( + 1, + 1, + &shard_dir, + cache, + LsmEngineConfig::default(), + RaftNodeOptions::default(), + handler, + watermarks, + clock, + &router, + ChannelPool::default(), + ) + .await; + match started { + Err(error) => { + let text = error.to_string(); + assert!( + text.contains("snapshot unrecoverable"), + "restart must fail closed before serving: {text}" + ); + assert!( + text.contains("not reconstructed"), + "an empty replacement LSM must not be treated as a new shard: {text}" + ); + } + Ok(group) => { + let latched = group.storage_unrecoverable().is_some(); + let rejected = group + .propose_local( + group.current_epoch(), + HlcTimestamp::ZERO, + codec::serialize(&BasicKvCommand::Put { + key: MetaKey::child(1, "fresh").encode(), + value: b"nope".to_vec(), + }) + .unwrap(), + ) + .await + .is_err(); + let _ = group.shutdown().await; + assert!( + latched && rejected, + "restart must be unrecoverable and must not accept a write onto an empty LSM" + ); + } + } +} + +#[tokio::test] +async fn restart_rebuilds_snapshot_from_lsm_without_a_prior_live_read() { + let dir = tempdir().unwrap(); + let shard_dir = dir.path().join("shards/00000000000000000001"); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let clock = Arc::new(HybridLogicalClock::new()); + { + let group = boot(&shard_dir, cache.clone(), clock.clone()).await; + let epoch = group.current_epoch(); + for name in ["alpha", "beta", "gamma"] { + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, name).encode(), + value: name.as_bytes().to_vec(), + }; + group + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + } + group.flush_snapshot_and_purge().await.unwrap(); + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + group.shutdown().await.unwrap(); + } + + let group = boot(&shard_dir, cache, clock).await; + let rewritten = fs::read_dir(shard_dir.join("raft/snapshots")) + .unwrap() + .flatten() + .any(|entry| { + let name = entry.file_name(); + let name = name.to_str().unwrap_or(""); + name.starts_with("snap_") && !name.contains('.') + }); + assert!( + rewritten, + "start must rewrite the snapshot from the LSM before a client read" + ); + assert!(group.storage_unrecoverable().is_none()); + assert_keys(&group).await; + group.shutdown().await.unwrap(); +} + +async fn spawn_transport(router: MultiRaftRouter) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let svc = RaftTransportServiceServer::new(MultiRaftTransportServer::new(router)); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(svc) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await; + }); + addr +} + +async fn start_voter( + node_id: u64, + shard_dir: &std::path::Path, + router: &MultiRaftRouter, + cache: Arc, +) -> Arc { + let watermarks = Arc::new(ShardWatermarkTracker::new(0, 0, 0)); + let handler = Arc::new(BasicKvApplyHandler::new( + ShardEpoch::initial(), + watermarks.clone(), + )); + ShardRaftGroup::start( + 1, + node_id, + shard_dir, + cache, + LsmEngineConfig::default(), + RaftNodeOptions { + heartbeat_interval_ms: 80, + election_timeout_min_ms: 250, + election_timeout_max_ms: 450, + ..RaftNodeOptions::default() + }, + handler, + watermarks, + Arc::new(HybridLogicalClock::new()), + router, + ChannelPool::default(), + ) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn learner_install_succeeds_after_leader_snapshot_file_deleted() { + let dir = tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let router1 = MultiRaftRouter::new(); + let router2 = MultiRaftRouter::new(); + let addr1 = spawn_transport(router1.clone()).await; + let addr2 = spawn_transport(router2.clone()).await; + let leader = start_voter(1, &dir.path().join("node1"), &router1, cache.clone()).await; + leader + .initialize_if_needed(BTreeMap::from([(1u64, addr1.clone())])) + .await + .unwrap(); + let epoch = leader.current_epoch(); + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, "alpha").encode(), + value: b"alpha".to_vec(), + }; + leader + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + let purged = leader.flush_snapshot_and_purge().await.unwrap(); + assert!(purged > 0); + let _ = fs::remove_dir_all(dir.path().join("node1/raft/snapshots")); + + let learner = start_voter(2, &dir.path().join("node2"), &router2, cache).await; + ConfChangeOrchestrator::add_learners_and_promote_voters( + leader.raft(), + &[(2, addr2)], + BTreeSet::from([1, 2]), + 64, + Duration::from_secs(10), + purged, + ) + .await + .expect("learner install must succeed after the leader rewrites its snapshot"); + let snap = learner + .state_machine() + .acquire_snapshot(learner.current_epoch(), purged, None) + .expect("installed learner state must be readable without being the leader"); + let key = MetaKey::child(1, "alpha").encode(); + assert_eq!( + learner.engine().get_with_snapshot(&key, &snap).unwrap(), + Some(b"alpha".to_vec()) + ); + leader.shutdown().await.unwrap(); + learner.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn learner_install_fails_closed_when_leader_is_unrecoverable() { + let dir = tempdir().unwrap(); + let cache = Arc::new(ShardedBlockCache::new(4 * 1024 * 1024)); + let router1 = MultiRaftRouter::new(); + let router2 = MultiRaftRouter::new(); + let addr1 = spawn_transport(router1.clone()).await; + let addr2 = spawn_transport(router2.clone()).await; + let leader = start_voter(1, &dir.path().join("node1"), &router1, cache.clone()).await; + leader + .initialize_if_needed(BTreeMap::from([(1u64, addr1)])) + .await + .unwrap(); + let epoch = leader.current_epoch(); + let cmd = BasicKvCommand::Put { + key: MetaKey::child(1, "alpha").encode(), + value: b"alpha".to_vec(), + }; + leader + .propose(epoch, HlcTimestamp::ZERO, codec::serialize(&cmd).unwrap()) + .await + .unwrap(); + leader.flush_snapshot_and_purge().await.unwrap(); + for entry in fs::read_dir(dir.path().join("node1/lsm")).unwrap() { + let entry = entry.unwrap(); + if entry.file_name().to_str().unwrap_or("").ends_with(".sst") { + fs::remove_file(entry.path()).unwrap(); + } + } + let _ = fs::remove_dir_all(dir.path().join("node1/raft/snapshots")); + let _learner = start_voter(2, &dir.path().join("node2"), &router2, cache).await; + let installed = ConfChangeOrchestrator::add_learners_and_promote_voters( + leader.raft(), + &[(2, addr2)], + BTreeSet::from([1, 2]), + 64, + Duration::from_secs(3), + 1, + ) + .await; + assert!( + installed.is_err(), + "an unrecoverable leader must not install a learner from a missing snapshot and LSM" + ); + let write = leader + .propose_local( + leader.current_epoch(), + HlcTimestamp::ZERO, + codec::serialize(&BasicKvCommand::Put { + key: MetaKey::child(1, "later").encode(), + value: b"no".to_vec(), + }) + .unwrap(), + ) + .await; + assert!( + write.is_err(), + "the leader must not keep accepting writes after it cannot rebuild the snapshot" + ); + leader.shutdown().await.ok(); +} diff --git a/crates/metadb-server/src/grpc_services.rs b/crates/metadb-server/src/grpc_services.rs index 49e0789..c972db5 100644 --- a/crates/metadb-server/src/grpc_services.rs +++ b/crates/metadb-server/src/grpc_services.rs @@ -1016,12 +1016,30 @@ impl ControlPlaneService for MetaDbGrpcService { }; let mut routes = Vec::new(); for cell in &cells { - routes.extend(cell.list_routes()); + match cell.routes_for_live_status() { + Ok(live) => routes.extend(live), + Err(error) => { + return Ok(Response::new(GetRoutesResponse { + error: Some(ErrorDetailProto::from(&error)), + shards: Vec::new(), + cluster_id: Vec::new(), + })); + } + } } (routes, cluster_id) } else if let Some(cell) = self.global_manager.get_cell(req.cell_id) { let cluster_id = cell.cluster_id(); - (cell.list_routes(), cluster_id) + match cell.routes_for_live_status() { + Ok(routes) => (routes, cluster_id), + Err(error) => { + return Ok(Response::new(GetRoutesResponse { + error: Some(ErrorDetailProto::from(&error)), + shards: Vec::new(), + cluster_id: Vec::new(), + })); + } + } } else { let e = MetaDbError::ShardRoutingError(format!("unknown cell {}", req.cell_id)); return Ok(Response::new(GetRoutesResponse { diff --git a/crates/metadb-server/src/lib.rs b/crates/metadb-server/src/lib.rs index 834d5a2..1d3ce75 100644 --- a/crates/metadb-server/src/lib.rs +++ b/crates/metadb-server/src/lib.rs @@ -858,6 +858,9 @@ impl PersistentMetaDbCluster { // `.raftlog` retention follows the 3-watermark gate in production (metadbd's loop). for cell in self.global_manager.all_cells() { for (_, grp, _) in cell.local_shard_handles() { + // Rewrite a missing snapshot from the LSM without waiting for a + // client read. A purged log has no other copy. + let _ = grp.repair_published_snapshot().await; let retained = grp.log_store().retained_entry_count(); if retained == 0 { continue; diff --git a/crates/metadb-server/tests/snapshot_status_tests.rs b/crates/metadb-server/tests/snapshot_status_tests.rs new file mode 100644 index 0000000..f6bacb0 --- /dev/null +++ b/crates/metadb-server/tests/snapshot_status_tests.rs @@ -0,0 +1,116 @@ +//! A missing snapshot must not leave cluster status advertising a live voter. + +use std::fs; + +use metadb_core::ROOT_INODE_ID; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; +use openraft::storage::RaftStateMachine; + +#[tokio::test] +async fn unrecoverable_snapshot_is_not_reported_as_a_live_voter() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.sharding.bootstrap_table_shards = false; + let (cluster, _address) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + cluster + .tx_router() + .mkdir(ROOT_INODE_ID, "bucket", 100, 1, 0o755) + .await + .unwrap(); + cluster + .tx_router() + .create_file(100, "alpha.dat", 1001, 7, 1, 0o644, 4) + .await + .unwrap(); + + let (_, group, _) = cluster.cell_manager().lookup_local_handle(1).unwrap(); + let purged = group.flush_snapshot_and_purge().await.unwrap(); + assert!(purged > 0); + let live = cluster.cell_manager().routes_for_live_status().unwrap(); + assert!( + live.iter() + .any(|route| { route.shard_id == 1 && route.peers.values().any(|(_, voter)| *voter) }), + "a healthy shard is still a live voter" + ); + + let shard_dir = cluster.cell_manager().host().shard_dir(1); + let mut removed_sst = 0usize; + for entry in fs::read_dir(shard_dir.join("lsm")).unwrap() { + let entry = entry.unwrap(); + if entry.file_name().to_str().unwrap_or("").ends_with(".sst") { + fs::remove_file(entry.path()).unwrap(); + removed_sst += 1; + } + } + assert!(removed_sst > 0, "flush did not create an SSTable"); + let _ = fs::remove_dir_all(shard_dir.join("raft/snapshots")); + + let mut state_machine = group.state_machine().clone(); + let error = state_machine.get_current_snapshot().await.unwrap_err(); + assert!( + error.to_string().contains("snapshot unrecoverable"), + "{error}" + ); + let status = cluster.cell_manager().routes_for_live_status().unwrap_err(); + let status = status.to_string(); + assert!( + status.contains("not a live voter"), + "cluster status must not keep advertising voters after the snapshot and LSM are gone: {status}" + ); + cluster.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn routes_for_live_status_detects_missing_snapshot_without_prior_get_current_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = dir.path().to_path_buf(); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.sharding.bootstrap_table_shards = false; + let (cluster, _address) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + cluster + .tx_router() + .mkdir(ROOT_INODE_ID, "bucket", 100, 1, 0o755) + .await + .unwrap(); + cluster + .tx_router() + .create_file(100, "alpha.dat", 1001, 7, 1, 0o644, 4) + .await + .unwrap(); + let (_, group, _) = cluster.cell_manager().lookup_local_handle(1).unwrap(); + group.flush_snapshot_and_purge().await.unwrap(); + let shard_dir = cluster.cell_manager().host().shard_dir(1); + let published = group.watermarks().snapshot_values().2; + let meta = shard_dir.join(format!("raft/snapshots/snap_{published:020}/SNAPSHOT_META")); + fs::remove_file(&meta).unwrap(); + assert!(!meta.is_file()); + + let live = cluster + .cell_manager() + .routes_for_live_status() + .expect("status must repair a missing snapshot from the LSM before advertising voters"); + assert!( + live.iter() + .any(|route| { route.shard_id == 1 && route.peers.values().any(|(_, voter)| *voter) }), + "a repaired shard remains a live voter" + ); + let rewritten = fs::read_dir(shard_dir.join("raft/snapshots")) + .unwrap() + .flatten() + .any(|entry| { + entry + .file_name() + .to_str() + .unwrap_or("") + .starts_with("snap_") + && !entry.file_name().to_str().unwrap_or("").contains('.') + }); + assert!( + rewritten, + "GetRoutes must not return voters while the published snapshot file is still gone" + ); + cluster.shutdown().await.unwrap(); +} diff --git a/docs/designs/correctness_hardening.md b/docs/designs/correctness_hardening.md index 544fce8..fdc4417 100644 --- a/docs/designs/correctness_hardening.md +++ b/docs/designs/correctness_hardening.md @@ -6,7 +6,7 @@ This change hardens acknowledged-write recovery, transaction migration, input va Recovery validates the entire unpurged committed log prefix, frame ordering and identity, and the durable vote/commit metadata before repairing files. Existing histories with missing `HARD_STATE` are rejected. A durable initialization marker also protects empty or fully purged stores from silently resetting their election identity. A fresh log has nothing to replay, so startup does not spend the recovery timeout before the first membership entry exists. -Snapshot installation requires a complete existing database image. It verifies required and duplicate filenames, the recovered checkpoint, LogId and membership, MANIFEST-to-SST metadata equality, and all data blocks before replacing live data. An empty or stale archive cannot create a new empty database and advance its advertised applied index. Snapshot publication syncs the temporary directory, renames it into place, and only then deletes the previous directory. A missing or unreadable snapshot directory is skipped. `get_current_snapshot` returns no snapshot instead of a fatal `Read Snapshot` error, and startup can rebuild one from the flushed LSM. Log purge stops at the newest complete snapshot directory, so a missing snapshot file does not also discard the log a restart still needs. +Snapshot installation requires a complete existing database image. It verifies required and duplicate filenames, the recovered checkpoint, LogId and membership, MANIFEST-to-SST metadata equality, and all data blocks before replacing live data. An empty or stale archive cannot create a new empty database and advance its advertised applied index. Snapshot publication syncs the temporary directory, renames it into place, and only then deletes the previous directory. A missing or unreadable snapshot directory is skipped. The segmented raft log is the write-ahead log only until it is purged. Before purge, a restart replays that log. After purge, the LSM is the remaining copy; there is no fourth command log. Startup and the maintenance tick rewrite a missing snapshot from the LSM before the process serves or advertises voters. If the flushed LSM cannot cover the purged index, open and cluster status fail closed with `snapshot unrecoverable`. A transient snapshot-publish I/O error does not latch that state. Committed data is not reconstructed when the purged log, the snapshot file, and the LSM are all gone. A state-machine operation lock serializes complete apply operations, checkpoint creation, snapshot replacement, and read-cut acquisition. This favors correctness and can pause writers while a snapshot is built. A later optimization should pin immutable state under a shorter lock and stream it without weakening the atomic checkpoint boundary. diff --git a/docs/en/architecture.md b/docs/en/architecture.md index 672d1f1..eb29e3a 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -95,7 +95,7 @@ Each local shard is an `openraft` 0.10 group. The log is a segmented CRC32C file | `max_in_snapshot_log_to_keep` | 0 | | log segment size | 64 MiB | -Log purge is gated by `min(state_flushed_index, aux_outbox_durable_index, last_snapshot_index)` and by the newest complete snapshot directory. The 250 ms maintenance loop calls `flush_snapshot_and_purge` when a shard still retains log entries, unless a snapshot watermark is already at least `state_flushed_index` and fewer than 64 entries remain. An operator backup hold clamps that purge, and snapshot-directory deletion, until the hold is released or its wall-clock expiry passes. See [Operations](./operations.md#backup-and-restore). +Log purge is gated by `min(state_flushed_index, aux_outbox_durable_index, last_snapshot_index)` and by the newest complete snapshot directory. The segmented log is the write-ahead log until that purge. After purge, the LSM is the only remaining copy. Startup and the 250 ms maintenance tick rewrite a missing snapshot from the LSM. If the flushed LSM cannot cover the purged prefix, open fails closed and cluster status does not report a live voter. There is no fourth copy of the command log. The 250 ms maintenance loop calls `flush_snapshot_and_purge` when a shard still retains log entries, unless a snapshot watermark is already at least `state_flushed_index` and fewer than 64 entries remain. An operator backup hold clamps that purge, and snapshot-directory deletion, until the hold is released or its wall-clock expiry passes. See [Operations](./operations.md#backup-and-restore). `ConfChangeOrchestrator::add_learners_and_promote_voters` adds learners, waits until replication lag is at most 64 entries (default timeout 10 s, the value `scale_out_replicas` passes), then calls `change_membership`. The in-process test `test_online_1_to_3_voter_snapshot_promotion_and_leader_failover` does this with three tonic listeners and a basic KV state machine inside one OS process. It does not start `metadbd`.