diff --git a/.github/workflows/security-adversarial-regressions.yml b/.github/workflows/security-adversarial-regressions.yml new file mode 100644 index 0000000..0091884 --- /dev/null +++ b/.github/workflows/security-adversarial-regressions.yml @@ -0,0 +1,56 @@ +name: Security adversarial regressions + +# Runs only the regression tests added for fail-closed restart, bearer +# comparison, uid, and restore refusal. A green run is not a first-ship pass, +# an official load, or a throughput result. + +on: + push: + branches: ["main", "**"] + tags: ["v*"] + pull_request: + branches: ["main"] + +concurrency: + group: security-adversarial-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + regressions: + name: Security and restore regression tests + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install Protobuf Compiler (protoc) + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Setup pinned Rust toolchain (rust-toolchain.toml, 1.100.0) + uses: dtolnay/rust-toolchain@1.100.0 + with: + components: rustfmt, clippy + + - name: Cache Cargo dependencies + uses: Swatinem/rust-cache@v2 + with: + key: ${{ runner.arch }}-security-adversarial + + - name: Run security adversarial regressions + run: cargo test -p metadb-server --test security_adversarial_regressions --locked -- --test-threads=1 + + - name: Run restore safety regressions + run: cargo test -p metadb-server --test restore_safety --locked -- --test-threads=1 diff --git a/crates/metadb-server/tests/restore_safety.rs b/crates/metadb-server/tests/restore_safety.rs new file mode 100644 index 0000000..7a641b3 --- /dev/null +++ b/crates/metadb-server/tests/restore_safety.rs @@ -0,0 +1,187 @@ +//! Restore checks that use the public backup API. +//! +//! A real artifact comes from `PersistentMetaDbCluster::create_backup`. Unsafe +//! manifest paths are written by editing that artifact's path field. `backup.rs` +//! stays unchanged, including the visibility of its manifest helpers. These tests +//! only check that restore refuses the case and does not write the destination. + +use std::fs; +use std::path::{Path, PathBuf}; + +use metadb_core::{append_crc32c_le, verify_crc32c_le, ROOT_INODE_ID}; +use metadb_raft::backup::{self, BACKUP_MANIFEST_FILE, BACKUP_TREE_DIR}; +use metadb_server::{PersistentMetaDbCluster, ServerTomlConfig}; + +async fn sample_artifact(root: &Path) -> PathBuf { + let data = root.join("data"); + let artifact = root.join("artifact"); + let mut cfg = ServerTomlConfig::default(); + cfg.node.data_dir = data; + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.security.shared_secret = "test-secret".into(); + let (cluster, _) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + cluster + .create_file_auto_id(ROOT_INODE_ID, "kept.dat", 1, 0o644, 8) + .await + .unwrap(); + cluster + .acquire_backup_hold(std::time::Duration::from_secs(120)) + .unwrap(); + cluster.create_backup(&artifact).unwrap(); + cluster.shutdown().await.unwrap(); + artifact +} + +fn copy_dir(source: &Path, dest: &Path) { + fs::create_dir_all(dest).unwrap(); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + let to = dest.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &to); + } else { + fs::copy(entry.path(), &to).unwrap(); + } + } +} + +fn tree_paths(artifact: &Path) -> Vec { + let tree = artifact.join(BACKUP_TREE_DIR); + let mut out = Vec::new(); + let mut stack = vec![tree.clone()]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + stack.push(path); + continue; + } + let relative = path.strip_prefix(&tree).unwrap(); + let mut text = String::new(); + for (index, component) in relative.components().enumerate() { + if index > 0 { + text.push('/'); + } + text.push_str(component.as_os_str().to_str().expect("utf-8 backup path")); + } + out.push(text); + } + } + out.sort(); + out +} + +fn path_field_start(payload: &[u8], path: &str) -> usize { + let bytes = path.as_bytes(); + let len_prefix = (bytes.len() as u64).to_le_bytes(); + let mut from = 0; + while from + bytes.len() <= payload.len() { + let Some(rel) = payload[from..] + .windows(bytes.len()) + .position(|window| window == bytes) + else { + break; + }; + let start = from + rel; + if start >= 8 && payload[start - 8..start] == len_prefix { + return start; + } + from = start + 1; + } + panic!("manifest has no length-prefixed path {path}"); +} + +fn replace_manifest_path(artifact: &Path, from: &str, to: &str) { + let manifest = artifact.join(BACKUP_MANIFEST_FILE); + let bytes = fs::read(&manifest).unwrap(); + let mut payload = verify_crc32c_le(&bytes, "backup record").unwrap().to_vec(); + let start = path_field_start(&payload, from); + let mut replacement = Vec::with_capacity(8 + to.len()); + replacement.extend_from_slice(&(to.len() as u64).to_le_bytes()); + replacement.extend_from_slice(to.as_bytes()); + payload.splice(start - 8..start + from.len(), replacement); + append_crc32c_le(&mut payload); + fs::write(manifest, payload).unwrap(); +} + +fn assert_directory_empty(dir: &Path) { + let names: Vec<_> = fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert!(names.is_empty(), "{} was written: {names:?}", dir.display()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn restore_rejects_non_empty_destination() { + let root = tempfile::tempdir().unwrap(); + let artifact = sample_artifact(root.path()).await; + let dest = root.path().join("dest"); + fs::create_dir_all(&dest).unwrap(); + let existing = dest.join("KEEP"); + fs::write(&existing, b"live-data").unwrap(); + + let error = backup::restore_backup(&artifact, &dest, &metadb_server::check_backup_tree) + .expect_err("restore into a non-empty destination must fail"); + assert!( + error.to_string().contains("is not empty"), + "non-empty destination error was: {error}" + ); + assert_eq!(fs::read(&existing).unwrap(), b"live-data"); + let mut names: Vec<_> = fs::read_dir(&dest) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + names.sort(); + assert_eq!(names, vec![std::ffi::OsString::from("KEEP")]); + assert!(!root.path().join(".dest.partial").exists()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn restore_rejects_unsafe_manifest_paths() { + let root = tempfile::tempdir().unwrap(); + let artifact = sample_artifact(root.path()).await; + let source_path = tree_paths(&artifact) + .into_iter() + .find(|path| path.starts_with("be/shards/")) + .expect("backup artifact contains a shard path"); + let absolute = root.path().join("absolute-probe"); + let absolute = absolute.to_str().expect("temp path is utf-8"); + let cases = [ + ("dotdot", "../outside"), + ("absolute", absolute), + ("backslash", r"windows\style"), + ("dot-component", "ok/./nested"), + ("dotdot-component", "ok/../up"), + ]; + + for (name, unsafe_path) in cases { + let case_artifact = root.path().join(format!("artifact-{name}")); + copy_dir(&artifact, &case_artifact); + replace_manifest_path(&case_artifact, &source_path, unsafe_path); + let scratch = root.path().join(format!("scratch-{name}")); + fs::create_dir_all(&scratch).unwrap(); + let dest = scratch.join("dest"); + let error = + backup::restore_backup(&case_artifact, &dest, &metadb_server::check_backup_tree) + .expect_err("unsafe manifest path must be rejected"); + let message = error.to_string(); + assert!( + message.contains("is not a relative backup path"), + "unsafe path {unsafe_path} was not rejected: {message}" + ); + assert!( + !dest.exists(), + "rejected restore created {}", + dest.display() + ); + assert_directory_empty(&scratch); + if Path::new(unsafe_path).is_absolute() { + assert!( + !Path::new(unsafe_path).exists(), + "rejected restore wrote {}", + unsafe_path + ); + } + } +} diff --git a/crates/metadb-server/tests/security_adversarial_regressions.rs b/crates/metadb-server/tests/security_adversarial_regressions.rs new file mode 100644 index 0000000..a60469c --- /dev/null +++ b/crates/metadb-server/tests/security_adversarial_regressions.rs @@ -0,0 +1,169 @@ +//! Defensive checks for fail-closed restart, bearer comparison, and `uid`. +//! +//! These tests observe current behavior. They do not change it, and they do not +//! try to bypass authentication. A green run of a residual-risk case is a record +//! of that gap, not a first-ship security pass. + +use metadb_core::ROOT_INODE_ID; +use metadb_proto::control_plane_service_client::ControlPlaneServiceClient; +use metadb_proto::metadata_service_client::MetadataServiceClient; +use metadb_proto::{CreateFileRequest, GetRoutesRequest, LookupRequest}; +use metadb_server::{BearerAuthClient, PersistentMetaDbCluster, ServerTomlConfig}; +use tonic::transport::Endpoint; +use tonic::Code; + +fn with_data_dir(mut cfg: ServerTomlConfig, dir: &std::path::Path) -> ServerTomlConfig { + cfg.node.data_dir = dir.to_path_buf(); + cfg +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn persisted_non_loopback_advertise_without_secret_fails_closed() { + // Starting again on 127.0.0.1 with an empty secret does not exercise this + // path. The data directory already saved the address from the first bind. + let dir = tempfile::tempdir().unwrap(); + let mut first = ServerTomlConfig::default(); + first = with_data_dir(first, dir.path()); + first.node.grpc_listen_addr = "0.0.0.0:0".into(); + first.security.shared_secret = "test-secret".into(); + let (cluster, grpc_addr) = PersistentMetaDbCluster::open_and_serve(first) + .await + .expect("first start with a secret must succeed"); + let persisted = grpc_addr.clone(); + cluster.shutdown().await.unwrap(); + assert!( + !persisted.starts_with("127.") + && !persisted.starts_with("[::1]") + && !persisted.to_ascii_lowercase().starts_with("localhost"), + "this case needs a persisted non-loopback advertise address, got {persisted}" + ); + + let mut restart = ServerTomlConfig::default(); + restart = with_data_dir(restart, dir.path()); + restart.node.grpc_listen_addr = "127.0.0.1:0".into(); + restart.security.shared_secret.clear(); + let err = match PersistentMetaDbCluster::open_and_serve(restart).await { + Ok((cluster, _)) => { + let _ = cluster.shutdown().await; + panic!("empty secret must fail closed when a saved advertise address is {persisted}"); + } + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("refusing to serve"), + "restart error was: {message}" + ); + assert!( + message.contains(&persisted) || message.contains("unauthenticated listener"), + "fail-closed error did not name the persisted bind: {message}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn loopback_whitespace_secret_is_not_treated_as_empty() { + // `reject_unauthenticated_bind` trims the secret, so a whitespace-only value + // still lets a loopback listener start. `BearerAuth` does not trim: it + // compares the secret as stored. A missing bearer is rejected, a different + // whitespace bearer is rejected, and only the literal whitespace bearer is + // accepted. This is a consistency gap. A green result is not a first-ship pass. + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg = with_data_dir(cfg, dir.path()); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.security.shared_secret = " ".into(); + let (cluster, grpc_addr) = PersistentMetaDbCluster::open_and_serve(cfg) + .await + .expect("loopback plus a whitespace secret still starts"); + let channel = Endpoint::from_shared(format!("http://{grpc_addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let missing = ControlPlaneServiceClient::new(channel.clone()) + .get_routes(GetRoutesRequest { cell_id: 0 }) + .await; + let mut other = + ControlPlaneServiceClient::with_interceptor(channel.clone(), BearerAuthClient::new(" ")); + let other = other.get_routes(GetRoutesRequest { cell_id: 0 }).await; + let mut literal = + ControlPlaneServiceClient::with_interceptor(channel, BearerAuthClient::new(" ")); + let accepted = literal.get_routes(GetRoutesRequest { cell_id: 0 }).await; + cluster.shutdown().await.unwrap(); + assert_eq!( + missing.unwrap_err().code(), + Code::Unauthenticated, + "a missing bearer must be rejected when the stored secret is whitespace" + ); + assert_eq!( + other.unwrap_err().code(), + Code::Unauthenticated, + "a different whitespace bearer must be rejected" + ); + assert!( + accepted.is_ok(), + "only the literal whitespace bearer should authenticate: {accepted:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn uid_is_not_an_authorization_boundary() { + // One valid bearer can create and look up files that carry different `uid` + // request fields. `uid` is stored metadata. It is not an identity or tenant + // boundary. A green result documents that gap and is not a first-ship pass. + let dir = tempfile::tempdir().unwrap(); + let mut cfg = ServerTomlConfig::default(); + cfg = with_data_dir(cfg, dir.path()); + cfg.node.grpc_listen_addr = "127.0.0.1:0".into(); + cfg.security.shared_secret = "test-secret".into(); + let (cluster, grpc_addr) = PersistentMetaDbCluster::open_and_serve(cfg).await.unwrap(); + let channel = Endpoint::from_shared(format!("http://{grpc_addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut meta = + MetadataServiceClient::with_interceptor(channel, BearerAuthClient::new("test-secret")); + + for (name, inode_id, fid, uid) in [ + ("one.dat", 1001u64, 5001u64, 1u64), + ("two.dat", 1002, 5002, 2), + ] { + let created = meta + .create_file(CreateFileRequest { + header: None, + pid: ROOT_INODE_ID, + name: name.into(), + inode_id, + fid, + uid, + mode: 0o644, + size_bytes: 8, + }) + .await + .expect("create_file rpc") + .into_inner(); + assert!( + created.error.is_none(), + "same bearer created {name} as uid {uid}: {:?}", + created.error + ); + } + + for (name, uid) in [("one.dat", 1u64), ("two.dat", 2u64)] { + let found = meta + .lookup(LookupRequest { + pid: ROOT_INODE_ID, + name: name.into(), + ..Default::default() + }) + .await + .expect("lookup rpc") + .into_inner(); + assert!(found.error.is_none(), "lookup {name}: {:?}", found.error); + assert!(found.found, "lookup did not find {name}"); + let attr = found.entry.expect("entry").attr.expect("attr"); + assert_eq!(attr.uid, uid, "{name} stored a different uid"); + } + cluster.shutdown().await.unwrap(); +}