diff --git a/crates/metadb-cluster/src/cell_manager.rs b/crates/metadb-cluster/src/cell_manager.rs index f7d8f86..9c7f611 100644 --- a/crates/metadb-cluster/src/cell_manager.rs +++ b/crates/metadb-cluster/src/cell_manager.rs @@ -1833,6 +1833,16 @@ impl DurableCellManager { } fn write_state_file(dir: &Path, st: &PersistedCellState) -> Result<()> { + let _shard_set = if dir + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("cell_")) + { + let root = dir.parent().unwrap_or_else(|| Path::new(".")); + Some(metadb_raft::backup::lock_be_shard_set(&root.join("be"))?) + } else { + None + }; let mut raw = codec::serialize(st).map_err(|e| MetaDbError::CodecError(e.to_string()))?; append_crc32c_le(&mut raw); let tmp = dir.join("CELL_MANAGER_STATE.tmp"); diff --git a/crates/metadb-cluster/src/split_worker.rs b/crates/metadb-cluster/src/split_worker.rs index 284041b..91fc63a 100644 --- a/crates/metadb-cluster/src/split_worker.rs +++ b/crates/metadb-cluster/src/split_worker.rs @@ -504,9 +504,12 @@ impl RaftShardSplitWorker { }, )?; drop(child_lsm); - fs::rename(&child_tmp_dir, &child_final_dir)?; - if let Some(parent_fs_dir) = child_final_dir.parent() { - sync_dir(parent_fs_dir)?; + let _shard_set = metadb_raft::backup::lock_be_shard_set(host.data_dir())?; + if !child_final_dir.exists() { + fs::rename(&child_tmp_dir, &child_final_dir)?; + if let Some(parent_fs_dir) = child_final_dir.parent() { + sync_dir(parent_fs_dir)?; + } } } Self::bail_crash(crash_at, SplitCrashPoint::AfterChildDirRename)?; diff --git a/crates/metadb-raft/src/backup.rs b/crates/metadb-raft/src/backup.rs index 9253acc..bc8032b 100644 --- a/crates/metadb-raft/src/backup.rs +++ b/crates/metadb-raft/src/backup.rs @@ -8,7 +8,7 @@ //! expires on the wall clock so a missed release cannot pin purge forever. use std::collections::{BTreeMap, BTreeSet}; -use std::fs::{self, OpenOptions}; +use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; @@ -78,6 +78,7 @@ pub const BACKUP_TREE_DIR: &str = "tree"; pub const BACKUP_JOB_FILE: &str = "BACKUP_JOB"; pub const BACKUP_JOB_RUNNING: &str = "BACKUP_JOB.running"; pub const BACKUP_JOB_RESULT: &str = "BACKUP_JOB_RESULT"; +pub const SHARD_SET_LOCK: &str = "SHARD_SET.lock"; pub const BACKUP_HOLD_VERSION: u32 = 1; pub const BACKUP_MANIFEST_VERSION: u32 = 1; pub const HOLD_PHASE_PENDING: u8 = 1; @@ -582,6 +583,40 @@ pub fn list_shard_ids(data_dir: &Path) -> Result> { Ok(ids) } +/// Exclusive lock covering shard-directory creation and registry publication. +/// +/// Backup holds it from the final shard-set check through the artifact rename so a +/// new shard cannot appear in that window and still be reported as a successful copy. +pub struct ShardSetLock { + _file: File, +} + +pub fn lock_be_shard_set(be_dir: &Path) -> Result { + fs::create_dir_all(be_dir)?; + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(be_dir.join(SHARD_SET_LOCK))?; + file.lock() + .map_err(|error| backup_error(format!("shard set lock: {error}")))?; + Ok(ShardSetLock { _file: file }) +} + +/// `be` directory when `shard_dir` is `/be/shards/`. +pub fn be_dir_for_shard(shard_dir: &Path) -> Option { + let shards = shard_dir.parent()?; + if shards.file_name()? != "shards" { + return None; + } + let be = shards.parent()?; + if be.file_name()? != "be" { + return None; + } + Some(be.to_path_buf()) +} + pub fn lsm_directory_is_locked(data_dir: &Path) -> bool { let Ok(entries) = fs::read_dir(data_dir.join("be").join("shards")) else { return false; @@ -864,24 +899,18 @@ fn backup_online_shard( let live_index = shard.engine.durable_checkpoint().flushed_raft_index; let (kind, image_index) = if live_index >= purged_now { // Hard-linked checkpoints must stay on the source filesystem. - let temp = data_dir.join(format!(".backup-ckpt-{:020}", shard.shard_id)); - if temp.exists() { - return Err(backup_error(format!( - "checkpoint scratch {} already exists; refusing to delete it", - temp.display() - ))); - } - let checkpoint = shard.engine.create_checkpoint(&temp).map_err(|error| { - let _ = fs::remove_dir_all(&temp); - backup_error(format!( - "shard {} checkpoint failed: {error}", - shard.shard_id - )) - })?; let lsm_rel = format!("be/shards/{:020}/lsm", shard.shard_id); let lsm_dest = tree.join(safe_relative(&lsm_rel).map_err(backup_error)?); - copy_dir_bytes(&temp, &lsm_dest)?; - let _ = fs::remove_dir_all(&temp); + let checkpoint = with_checkpoint_scratch(data_dir, shard.shard_id, |temp| { + let checkpoint = shard.engine.create_checkpoint(temp).map_err(|error| { + backup_error(format!( + "shard {} checkpoint failed: {error}", + shard.shard_id + )) + })?; + copy_dir_bytes(temp, &lsm_dest)?; + Ok(checkpoint) + })?; record_tree_files(&lsm_dest, &lsm_rel, files)?; (IMAGE_FLUSHED_LSM, checkpoint.flushed_raft_index) } else { @@ -947,9 +976,7 @@ fn backup_stopped_shard( copy_protected_snapshot(&raft_dir, floor, &relative_raft, tree, files)?; let lsm_dir = shard_directory.join("lsm"); let live_index = if lsm_dir.join("CURRENT").is_file() { - let (manifest, recovered) = ManifestLog::open_existing(&lsm_dir)?; - drop(manifest); - recovered.checkpoint.flushed_raft_index + ManifestLog::read_checkpoint_readonly(&lsm_dir)?.flushed_raft_index } else { 0 }; @@ -1172,8 +1199,7 @@ fn finish_manifest( } let parent = destination_parent(destination); sync_directory_and_ancestors(&parent)?; - fs::rename(staging, destination)?; - staging_guard.disarm(); + publish_staging_under_shard_lock(data_dir, &copied, staging, destination, staging_guard)?; sync_dir(&parent)?; Ok(BackupSummary { hold_id: manifest.hold_id, @@ -1183,6 +1209,117 @@ fn finish_manifest( }) } +#[cfg(test)] +static BEFORE_LOCKED_RENAME: Mutex> = Mutex::new(None); + +fn publish_staging_under_shard_lock( + data_dir: &Path, + copied: &BTreeSet, + staging: &Path, + destination: &Path, + staging_guard: &mut Staging, +) -> Result<()> { + let _shard_set = lock_be_shard_set(&data_dir.join("be"))?; + reject_uncovered_live_shards(data_dir, copied)?; + #[cfg(test)] + { + if let Some(hook) = BEFORE_LOCKED_RENAME.lock().unwrap().take() { + hook(data_dir); + } + } + reject_uncovered_live_shards(data_dir, copied)?; + fs::rename(staging, destination)?; + staging_guard.disarm(); + Ok(()) +} + +fn reject_uncovered_live_shards(data_dir: &Path, copied: &BTreeSet) -> Result<()> { + for shard_id in list_shard_ids(data_dir)? { + if !copied.contains(&shard_id) { + return Err(backup_error(format!( + "shard {shard_id} appeared during the backup and is not in the artifact" + ))); + } + } + let registry = data_dir.join("be").join("NODE_SHARD_REGISTRY"); + if registry.is_file() { + for shard_id in crate::group_host::shard_ids_in_registry_file(®istry)? { + if !copied.contains(&shard_id) { + return Err(backup_error(format!( + "NODE_SHARD_REGISTRY references shard {shard_id} which was not copied" + ))); + } + } + } + Ok(()) +} + +fn checkpoint_scratch_name(shard_id: ShardId) -> String { + format!(".backup-ckpt-{shard_id:020}") +} + +fn checkpoint_scratch_lock_name(shard_id: ShardId) -> String { + format!(".backup-ckpt-{shard_id:020}.lock") +} + +struct CheckpointScratchGuard { + _file: File, +} + +fn lock_checkpoint_scratch(data_dir: &Path, shard_id: ShardId) -> Result { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(data_dir.join(checkpoint_scratch_lock_name(shard_id)))?; + file.lock().map_err(|error| { + backup_error(format!( + "checkpoint scratch lock for shard {shard_id}: {error}" + )) + })?; + Ok(CheckpointScratchGuard { _file: file }) +} + +/// Clears crash residue, then runs `body` while this process still owns the scratch. +/// Another backup of the same shard waits instead of deleting the directory mid-copy. +fn with_checkpoint_scratch( + data_dir: &Path, + shard_id: ShardId, + body: impl FnOnce(&Path) -> Result, +) -> Result { + let _guard = lock_checkpoint_scratch(data_dir, shard_id)?; + clear_checkpoint_scratch(data_dir, shard_id)?; + let path = data_dir.join(checkpoint_scratch_name(shard_id)); + let result = body(&path); + if path.exists() { + if let Err(error) = clear_checkpoint_scratch(data_dir, shard_id) { + if result.is_ok() { + return Err(error); + } + } + } + result +} + +/// Removes one crash scratch directory. A symlink or a non-directory is left in place. +fn clear_checkpoint_scratch(data_dir: &Path, shard_id: ShardId) -> Result<()> { + let path = data_dir.join(checkpoint_scratch_name(shard_id)); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(backup_error(format!( + "checkpoint scratch {} is not a plain directory; refusing to delete it", + path.display() + ))); + } + fs::remove_dir_all(&path)?; + Ok(()) +} + fn ensure_registry_matches(root: &Path, copied: &BTreeSet) -> Result<()> { let registry = root.join("be").join("NODE_SHARD_REGISTRY"); let referenced = crate::group_host::shard_ids_in_registry_file(®istry)?; @@ -1831,6 +1968,170 @@ mod tests { assert!(error.to_string().contains("shard 99"), "{error}"); } + #[test] + fn offline_backup_without_a_hold_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("be").join("shards")).unwrap(); + let destination = dir.path().join("out"); + let error = create_offline_backup(dir.path(), &destination, &|_| Ok(())).unwrap_err(); + assert!( + error + .to_string() + .contains("install a hold before copying so purge cannot remove the image"), + "{error}" + ); + assert!(!destination.exists()); + } + + #[test] + fn shard_directory_cannot_appear_while_the_publish_lock_is_held() { + let dir = tempfile::tempdir().unwrap(); + let be = dir.path().join("be"); + fs::create_dir_all(be.join("shards")).unwrap(); + let guard = lock_be_shard_set(&be).unwrap(); + let started = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let created = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let be_thread = be.clone(); + let started_thread = started.clone(); + let created_thread = created.clone(); + let handle = thread::spawn(move || { + started_thread.store(true, std::sync::atomic::Ordering::Release); + let shard = be_thread.join("shards").join(format!("{:020}", 99u64)); + if let Some(parent) = be_dir_for_shard(&shard) { + let _held = lock_be_shard_set(&parent).unwrap(); + fs::create_dir_all(&shard).unwrap(); + } + created_thread.store(true, std::sync::atomic::Ordering::Release); + }); + while !started.load(std::sync::atomic::Ordering::Acquire) { + thread::yield_now(); + } + thread::sleep(Duration::from_millis(50)); + assert!(!created.load(std::sync::atomic::Ordering::Acquire)); + assert!(!be.join("shards").join(format!("{:020}", 99u64)).exists()); + drop(guard); + handle.join().unwrap(); + assert!(created.load(std::sync::atomic::Ordering::Acquire)); + assert!(be.join("shards").join(format!("{:020}", 99u64)).is_dir()); + } + + #[test] + fn a_shard_created_before_rename_rejects_publication() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all( + dir.path() + .join("be") + .join("shards") + .join(format!("{:020}", 1u64)), + ) + .unwrap(); + let staging_parent = dir.path().join("stage-parent"); + fs::create_dir_all(&staging_parent).unwrap(); + let destination = staging_parent.join("artifact"); + let mut staging = Staging::create(&destination).unwrap(); + fs::write(staging.path.join("marker"), b"ok").unwrap(); + *BEFORE_LOCKED_RENAME.lock().unwrap() = Some(|data_dir| { + fs::create_dir_all( + data_dir + .join("be") + .join("shards") + .join(format!("{:020}", 99u64)), + ) + .unwrap(); + }); + let copied = BTreeSet::from([1u64]); + let staging_path = staging.path.clone(); + let error = publish_staging_under_shard_lock( + dir.path(), + &copied, + &staging_path, + &destination, + &mut staging, + ) + .unwrap_err(); + assert!(error.to_string().contains("shard 99"), "{error}"); + assert!(!destination.exists()); + assert!(dir + .path() + .join("be") + .join("shards") + .join(format!("{:020}", 99u64)) + .is_dir()); + } + + #[test] + fn checkpoint_scratch_is_not_cleared_while_another_backup_holds_it() { + let dir = tempfile::tempdir().unwrap(); + let scratch = dir.path().join(checkpoint_scratch_name(1)); + let release = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let started = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let cleared = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let dir_holder = dir.path().to_path_buf(); + let release_holder = release.clone(); + let started_holder = started.clone(); + let holder = thread::spawn(move || { + let guard = lock_checkpoint_scratch(&dir_holder, 1).unwrap(); + fs::create_dir_all(&scratch).unwrap(); + fs::write(scratch.join("KEEP"), b"live").unwrap(); + started_holder.store(true, std::sync::atomic::Ordering::Release); + while !release_holder.load(std::sync::atomic::Ordering::Acquire) { + thread::yield_now(); + } + drop(guard); + }); + while !started.load(std::sync::atomic::Ordering::Acquire) { + thread::yield_now(); + } + let dir_worker = dir.path().to_path_buf(); + let cleared_worker = cleared.clone(); + let worker = thread::spawn(move || { + with_checkpoint_scratch(&dir_worker, 1, |_path| { + cleared_worker.store(true, std::sync::atomic::Ordering::Release); + Ok(()) + }) + .unwrap(); + }); + thread::sleep(Duration::from_millis(100)); + assert!( + dir.path() + .join(checkpoint_scratch_name(1)) + .join("KEEP") + .is_file(), + "the waiting backup removed a scratch that was still in use" + ); + assert!(!cleared.load(std::sync::atomic::Ordering::Acquire)); + release.store(true, std::sync::atomic::Ordering::Release); + holder.join().unwrap(); + worker.join().unwrap(); + assert!(cleared.load(std::sync::atomic::Ordering::Acquire)); + assert!(!dir.path().join(checkpoint_scratch_name(1)).exists()); + } + + #[test] + fn crash_checkpoint_scratch_is_removed_without_touching_other_names() { + let dir = tempfile::tempdir().unwrap(); + let scratch = dir.path().join(checkpoint_scratch_name(1)); + fs::create_dir_all(scratch.join("nested")).unwrap(); + fs::write(scratch.join("nested").join("marker"), b"old").unwrap(); + let decoy = dir.path().join(".backup-ckpt-0001"); + fs::create_dir_all(&decoy).unwrap(); + fs::write(decoy.join("keep"), b"keep").unwrap(); + let keep_dir = dir.path().join("keep-me"); + fs::create_dir_all(&keep_dir).unwrap(); + clear_checkpoint_scratch(dir.path(), 1).unwrap(); + assert!(!scratch.exists()); + assert_eq!(fs::read(decoy.join("keep")).unwrap(), b"keep"); + assert!(keep_dir.is_dir()); + + let target = dir.path().join("real-data"); + fs::create_dir_all(&target).unwrap(); + fs::write(target.join("KEEP"), b"live").unwrap(); + std::os::unix::fs::symlink(&target, dir.path().join(checkpoint_scratch_name(2))).unwrap(); + let error = clear_checkpoint_scratch(dir.path(), 2).unwrap_err(); + assert!(error.to_string().contains("refusing to delete"), "{error}"); + assert_eq!(fs::read(target.join("KEEP")).unwrap(), b"live"); + } + #[test] fn completing_a_job_leaves_a_newer_request_in_place() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/metadb-raft/src/group_host.rs b/crates/metadb-raft/src/group_host.rs index b3dd036..222e77c 100644 --- a/crates/metadb-raft/src/group_host.rs +++ b/crates/metadb-raft/src/group_host.rs @@ -933,6 +933,7 @@ impl ShardGroupHost { shard_id: ShardId, state: RegistryShardState, ) -> Result<()> { + let _shard_set = crate::backup::lock_be_shard_set(&self.data_dir)?; let mut guard = self.registry.write().unwrap(); guard.insert(shard_id, state); Self::persist_registry(&self.data_dir, &guard) diff --git a/crates/metadb-raft/src/state_machine.rs b/crates/metadb-raft/src/state_machine.rs index 52306b6..14a966c 100644 --- a/crates/metadb-raft/src/state_machine.rs +++ b/crates/metadb-raft/src/state_machine.rs @@ -693,7 +693,12 @@ impl ShardStateMachine { watermarks: Arc, ) -> Result<(Self, watch::Receiver)> { let shard_dir = shard_dir.as_ref().to_path_buf(); - fs::create_dir_all(&shard_dir)?; + if let Some(be_dir) = crate::backup::be_dir_for_shard(&shard_dir) { + let _shard_set = crate::backup::lock_be_shard_set(&be_dir)?; + fs::create_dir_all(&shard_dir)?; + } else { + fs::create_dir_all(&shard_dir)?; + } // Recover interrupted `lsm_installing_*` or `lsm_old_*` swap if power failed mid-install. let lsm_dir = shard_dir.join("lsm"); diff --git a/crates/metadb-server/tests/backup_restore.rs b/crates/metadb-server/tests/backup_restore.rs index b8786ba..8b1bd54 100644 --- a/crates/metadb-server/tests/backup_restore.rs +++ b/crates/metadb-server/tests/backup_restore.rs @@ -283,7 +283,13 @@ fn set_files_readonly(root: &Path, readonly: bool) { continue; } let mut permissions = std::fs::metadata(¤t).unwrap().permissions(); - permissions.set_readonly(readonly); + let mode = std::os::unix::fs::PermissionsExt::mode(&permissions); + let mode = if readonly { + mode & !0o222 + } else { + mode | 0o200 + }; + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, mode); std::fs::set_permissions(¤t, permissions).unwrap(); } } @@ -337,6 +343,63 @@ fn copy_tree(source: &Path, dest: &Path) { assert!(status.success()); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn offline_backup_does_not_mutate_source_manifests() { + let dir = tempdir().unwrap(); + let cfg = persistent_cfg(dir.path()); + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + cluster + .create_file_auto_id(ROOT_INODE_ID, "offline.dat", 4, 0o644, 8) + .await + .unwrap(); + cluster.run_maintenance_tick().await.unwrap(); + cluster + .acquire_backup_hold(Duration::from_secs(120)) + .unwrap(); + cluster.shutdown().await.unwrap(); + + let manifests = walk_files(dir.path()) + .into_iter() + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("MANIFEST-")) + }) + .collect::>(); + assert!( + !manifests.is_empty(), + "maintenance did not write a MANIFEST" + ); + let before = manifests + .iter() + .map(|path| { + let metadata = std::fs::metadata(path).unwrap(); + ( + path.clone(), + std::fs::read(path).unwrap(), + std::os::unix::fs::PermissionsExt::mode(&metadata.permissions()), + ) + }) + .collect::>(); + for (path, _, mode) in &before { + let mut permissions = std::fs::metadata(path).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, mode & !0o222); + std::fs::set_permissions(path, permissions).unwrap(); + } + let outside = tempdir().unwrap(); + let output = outside.path().join("offline-artifact"); + let result = + backup::create_offline_backup(dir.path(), &output, &metadb_server::check_backup_tree); + for (path, bytes, mode) in &before { + let mut permissions = std::fs::metadata(path).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, *mode); + std::fs::set_permissions(path, permissions).unwrap(); + assert_eq!(std::fs::read(path).unwrap(), *bytes); + } + result.expect("a read-only source MANIFEST must still produce an offline backup"); + assert!(output.join("BACKUP_MANIFEST").is_file()); +} + fn walk_files(root: &Path) -> Vec { let mut out = Vec::new(); let mut stack = vec![root.to_path_buf()]; diff --git a/crates/metadb-storage/src/manifest.rs b/crates/metadb-storage/src/manifest.rs index 7b01df8..746d178 100644 --- a/crates/metadb-storage/src/manifest.rs +++ b/crates/metadb-storage/src/manifest.rs @@ -309,6 +309,32 @@ impl ManifestLog { Self::open_internal(lsm_dir, false) } + /// Reads the flushed checkpoint without opening `MANIFEST` for append or repairing it. + /// + /// Offline backup uses this so a read of the source image cannot change the live file. + pub fn read_checkpoint_readonly(lsm_dir: &Path) -> Result { + let current_path = lsm_dir.join("CURRENT"); + let manifest_name = fs::read_to_string(¤t_path)?.trim().to_string(); + if manifest_name.contains('/') + || manifest_name.contains('\\') + || manifest_name.contains("..") + { + return Err(MetaDbError::ManifestCorrupted(format!( + "invalid CURRENT content '{manifest_name}'" + ))); + } + let manifest_path = lsm_dir.join(&manifest_name); + let mut raw = Vec::new(); + File::open(&manifest_path)?.read_to_end(&mut raw)?; + let (state, torn) = Self::replay_manifest_bytes(&raw)?; + if torn.is_some() { + return Err(MetaDbError::ManifestCorrupted( + "incomplete MANIFEST in existing checkpoint".to_string(), + )); + } + Ok(state.checkpoint) + } + fn open_internal(lsm_dir: &Path, recover: bool) -> Result<(Self, RecoveredManifestState)> { if recover { fs::create_dir_all(lsm_dir)?; @@ -385,11 +411,57 @@ impl ManifestLog { )); } + let (recovered, truncate_to) = Self::replay_manifest_bytes(&raw)?; + let next_file_id = recovered.next_file_id; + let next_flush_generation = recovered.next_flush_generation; + let checkpoint = recovered.checkpoint; + let live_files = recovered.live_files; + + let file = OpenOptions::new() + .read(true) + .append(true) + .open(&manifest_path)?; + if let Some(valid_len) = truncate_to { + if !recover { + return Err(MetaDbError::ManifestCorrupted( + "incomplete MANIFEST in existing checkpoint".to_string(), + )); + } + let trunc_file = OpenOptions::new().write(true).open(&manifest_path)?; + trunc_file.set_len(valid_len as u64)?; + trunc_file.sync_data()?; + } + + if recover { + Self::cleanup_orphan_files(lsm_dir, &live_files, &manifest_name)?; + } + + Ok(( + Self { + lsm_dir: lsm_dir.to_path_buf(), + manifest_seq: seq, + manifest_path, + file, + }, + RecoveredManifestState { + next_file_id, + next_flush_generation, + checkpoint, + live_files, + }, + )) + } + + fn replay_manifest_bytes(raw: &[u8]) -> Result<(RecoveredManifestState, Option)> { + if raw.is_empty() { + return Err(MetaDbError::ManifestCorrupted( + "MANIFEST has no baseline record".to_string(), + )); + } let mut next_file_id = 1u64; let mut next_flush_generation = 1u64; let mut checkpoint = DurableShardCheckpoint::default(); let mut live_files: BTreeMap<(u8, u64), SstFileMeta> = BTreeMap::new(); - let mut offset = 0usize; let mut truncate_to: Option = None; @@ -472,42 +544,17 @@ impl ManifestLog { next_flush_generation = next_flush_generation.max(following_generation); live_files.insert((add.level, add.file_id), add); } - offset = record_end; } - let file = OpenOptions::new() - .read(true) - .append(true) - .open(&manifest_path)?; - if let Some(valid_len) = truncate_to { - if !recover { - return Err(MetaDbError::ManifestCorrupted( - "incomplete MANIFEST in existing checkpoint".to_string(), - )); - } - let trunc_file = OpenOptions::new().write(true).open(&manifest_path)?; - trunc_file.set_len(valid_len as u64)?; - trunc_file.sync_data()?; - } - - if recover { - Self::cleanup_orphan_files(lsm_dir, &live_files, &manifest_name)?; - } - Ok(( - Self { - lsm_dir: lsm_dir.to_path_buf(), - manifest_seq: seq, - manifest_path, - file, - }, RecoveredManifestState { next_file_id, next_flush_generation, checkpoint, live_files, }, + truncate_to, )) } diff --git a/docs/en/operations.md b/docs/en/operations.md index 349c89b..eaf005e 100644 --- a/docs/en/operations.md +++ b/docs/en/operations.md @@ -149,7 +149,7 @@ Variants are defined in `crates/metadb-core/src/error.rs`. gRPC methods usually This is a single-node operator path for one data directory. It is not a multi-Cell or Jumbo backup, and it is not a production backup system. There is no independent Raft WAL. If the segmented log, the complete snapshot, and the flushed LSM image are all missing, committed keys are not reconstructed. A passing test run does not make this path a first-ship pass. -The restorable image is the newest flushed LSM checkpoint, or the complete snapshot the hold protects when that checkpoint does not cover the purged log prefix, plus the segmented Raft log that is still on disk. Keys that are only in an unflushed memtable are restored by replaying that log. The copy does not rename or truncate files in the live data directory. +The restorable image is the newest flushed LSM checkpoint, or the complete snapshot the hold protects when that checkpoint does not cover the purged log prefix, plus the segmented Raft log that is still on disk. The cell directory's `CLUSTER_IDENTITY` is copied with that image so the restored process keeps the same node instance id. Keys that are only in an unflushed memtable are restored by replaying that log. The copy does not rename or truncate files in the live data directory. ### Hold lifetime @@ -176,6 +176,8 @@ metadb-ctl backup restore --backup /var/backups/metadb-2026-11-19 --data-dir /va metadbd --config restored.toml ``` +Publication holds `be/SHARD_SET.lock` from the final shard-directory and registry check through the artifact rename. Shard directory creation, split publication, registry updates, and cell-state writes take that same lock, so a shard that becomes visible in that window cannot be omitted from a backup that still reports success. A directory or registry entry that appears before the locked check fails the backup instead. Offline backup reads each source `MANIFEST` without opening it for append. A crash scratch directory named exactly `.backup-ckpt-` plus a 20-digit shard id is removed before the next online checkpoint on that shard. The cleanup, checkpoint, byte copy, and removal share `.backup-ckpt-.lock`, so one backup cannot delete another backup's live scratch. A symlink or any other name is left in place. + `backup create` fails when no active hold is installed, when a shard is not listed on that hold, when the log is purged past the hold floor, or when neither the flushed LSM nor a protected complete snapshot covers the purged prefix. The artifact contains `BACKUP_MANIFEST` and `tree/`. Restore checks the manifest version, every file length and CRC32C, the Raft log (a torn tail is rejected rather than repaired), and that each shard LSM checkpoint matches the manifest and covers the purged index. It does not invent keys. After restore, open the new directory with the same node id and cell id. Look up and list the keys that had been committed. Keys that were never flushed and were not in the copied log are not recovered.